kopia lustrzana https://github.com/onthegomap/planetiler
Fix feature merge consistent ordering (#789)
rodzic
062528b1ee
commit
14b217d6f6
|
@ -0,0 +1,57 @@
|
||||||
|
package com.onthegomap.planetiler.collection;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class StressTestKWayMerge {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws InterruptedException {
|
||||||
|
for (int i = 1; i < 20; i++) {
|
||||||
|
test(i, 100_000, 200_000);
|
||||||
|
}
|
||||||
|
for (int i = 50; i <= 500; i += 50) {
|
||||||
|
test(i, 10_000, 20_000);
|
||||||
|
}
|
||||||
|
test(5_000, 1000, 2000);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void test(int n, long items, long maxKey) throws InterruptedException {
|
||||||
|
System.out.println("test(" + n + ")");
|
||||||
|
var random = new Random(0);
|
||||||
|
List<List<SortableFeature>> featureLists = new ArrayList<>();
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
List<SortableFeature> list = new ArrayList<>();
|
||||||
|
featureLists.add(list);
|
||||||
|
for (int j = 0; j < items; j++) {
|
||||||
|
byte[] bytes = new byte[random.nextInt(1, 10)];
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
list.add(new SortableFeature(random.nextLong(maxKey), bytes));
|
||||||
|
}
|
||||||
|
executorService.submit(() -> list.sort(Comparator.naturalOrder()));
|
||||||
|
}
|
||||||
|
executorService.shutdown();
|
||||||
|
executorService.awaitTermination(1, TimeUnit.DAYS);
|
||||||
|
|
||||||
|
|
||||||
|
var iter =
|
||||||
|
LongMerger.mergeIterators(featureLists.stream().map(List::iterator).toList(), SortableFeature.COMPARE_BYTES);
|
||||||
|
var last = iter.next();
|
||||||
|
int i = 1;
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
i++;
|
||||||
|
var item = iter.next();
|
||||||
|
if (last.compareTo(item) > 0) {
|
||||||
|
System.err
|
||||||
|
.println("items out of order lists=" + n + " last=" + last + " item=" + item + " i=" + i);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
last = item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -88,7 +88,8 @@ public class TileArchiveWriter {
|
||||||
TileArchiveMetadata tileArchiveMetadata, Path layerStatsPath, PlanetilerConfig config, Stats stats) {
|
TileArchiveMetadata tileArchiveMetadata, Path layerStatsPath, PlanetilerConfig config, Stats stats) {
|
||||||
var timer = stats.startStage("archive");
|
var timer = stats.startStage("archive");
|
||||||
|
|
||||||
int readThreads = config.featureReadThreads();
|
int chunksToRead = Math.max(1, features.chunksToRead());
|
||||||
|
int readThreads = Math.min(config.featureReadThreads(), chunksToRead);
|
||||||
int threads = config.threads();
|
int threads = config.threads();
|
||||||
int processThreads = threads < 10 ? threads : threads - readThreads;
|
int processThreads = threads < 10 ? threads : threads - readThreads;
|
||||||
int tileWriteThreads = config.tileWriteThreads();
|
int tileWriteThreads = config.tileWriteThreads();
|
||||||
|
|
|
@ -213,7 +213,7 @@ class ArrayLongMinHeap implements LongMinHeap {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (comparePosPos(value, minValue, pos, minChild) <= 0) {
|
if (compareIdPos(value, minValue, id, minChild) <= 0) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
posToValue[pos] = minValue;
|
posToValue[pos] = minValue;
|
||||||
|
|
|
@ -256,6 +256,11 @@ class ExternalMergeSort implements FeatureSort {
|
||||||
return LongMerger.mergeIterators(iterators, SortableFeature.COMPARE_BYTES);
|
return LongMerger.mergeIterators(iterators, SortableFeature.COMPARE_BYTES);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int chunksToRead() {
|
||||||
|
return chunks.size();
|
||||||
|
}
|
||||||
|
|
||||||
public int chunks() {
|
public int chunks() {
|
||||||
return chunks.size();
|
return chunks.size();
|
||||||
}
|
}
|
||||||
|
|
|
@ -314,6 +314,10 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public int chunksToRead() {
|
||||||
|
return sorter.chunksToRead();
|
||||||
|
}
|
||||||
|
|
||||||
public interface RenderedFeatureEncoder extends Function<RenderedFeature, SortableFeature>, Closeable {}
|
public interface RenderedFeatureEncoder extends Function<RenderedFeature, SortableFeature>, Closeable {}
|
||||||
|
|
||||||
public record Reader(Worker readWorker, Iterable<TileFeatures> result) {}
|
public record Reader(Worker readWorker, Iterable<TileFeatures> result) {}
|
||||||
|
|
|
@ -74,6 +74,11 @@ interface FeatureSort extends Iterable<SortableFeature>, DiskBacked, MemoryEstim
|
||||||
.mapToObj(list::get)
|
.mapToObj(list::get)
|
||||||
.iterator();
|
.iterator();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int chunksToRead() {
|
||||||
|
return list.size();
|
||||||
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,6 +139,8 @@ interface FeatureSort extends Iterable<SortableFeature>, DiskBacked, MemoryEstim
|
||||||
return new ParallelIterator(reader, LongMerger.mergeSuppliers(queues, SortableFeature.COMPARE_BYTES));
|
return new ParallelIterator(reader, LongMerger.mergeSuppliers(queues, SortableFeature.COMPARE_BYTES));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int chunksToRead();
|
||||||
|
|
||||||
record ParallelIterator(Worker reader, @Override Iterator<SortableFeature> iterator)
|
record ParallelIterator(Worker reader, @Override Iterator<SortableFeature> iterator)
|
||||||
implements Iterable<SortableFeature> {}
|
implements Iterable<SortableFeature> {}
|
||||||
}
|
}
|
||||||
|
|
|
@ -230,8 +230,8 @@ public class LongMerger {
|
||||||
items[id] = next;
|
items[id] = next;
|
||||||
heap.updateHead(next.key());
|
heap.updateHead(next.key());
|
||||||
} else {
|
} else {
|
||||||
items[id] = null;
|
|
||||||
heap.poll();
|
heap.poll();
|
||||||
|
items[id] = null;
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,11 +2,13 @@ package com.onthegomap.planetiler.collection;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.NoSuchElementException;
|
import java.util.NoSuchElementException;
|
||||||
|
import java.util.Random;
|
||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
import java.util.stream.LongStream;
|
import java.util.stream.LongStream;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
@ -199,6 +201,37 @@ class LongMergerTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 100})
|
||||||
|
void stressTest(int n) {
|
||||||
|
int items = 10;
|
||||||
|
int maxKey = 20;
|
||||||
|
var random = new Random(0);
|
||||||
|
List<List<SortableFeature>> featureLists = new ArrayList<>();
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
List<SortableFeature> list = new ArrayList<>();
|
||||||
|
featureLists.add(list);
|
||||||
|
for (int j = 0; j < items; j++) {
|
||||||
|
byte[] bytes = new byte[]{(byte) random.nextInt(256)};
|
||||||
|
random.nextBytes(bytes);
|
||||||
|
list.add(new SortableFeature(random.nextLong(maxKey), bytes));
|
||||||
|
}
|
||||||
|
list.sort(Comparator.naturalOrder());
|
||||||
|
}
|
||||||
|
|
||||||
|
var iter =
|
||||||
|
LongMerger.mergeIterators(featureLists.stream().map(List::iterator).toList(), SortableFeature.COMPARE_BYTES);
|
||||||
|
var last = iter.next();
|
||||||
|
int i = 1;
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
i++;
|
||||||
|
var item = iter.next();
|
||||||
|
assertTrue(last.compareTo(item) <= 0,
|
||||||
|
"items out of order i=" + i + " lists=" + n + " last=" + last + " item=" + item);
|
||||||
|
last = item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static long[] parse(String in) {
|
private static long[] parse(String in) {
|
||||||
return in == null ? new long[0] : Stream.of(in.split("\\s+"))
|
return in == null ? new long[0] : Stream.of(in.split("\\s+"))
|
||||||
.map(String::strip)
|
.map(String::strip)
|
||||||
|
|
|
@ -81,21 +81,25 @@ class LongMinHeapTest {
|
||||||
|
|
||||||
@ParameterizedTest
|
@ParameterizedTest
|
||||||
@CsvSource({
|
@CsvSource({
|
||||||
"0, 1, 2, 3, 4, 5",
|
"0, 1, 2, 3, 4, 5, 6, 7",
|
||||||
"5, 4, 3, 2, 1, 0",
|
"7, 6, 5, 4, 3, 2, 1, 0",
|
||||||
"0, 1, 2, 5, 4, 3",
|
"0, 1, 2, 6, 7, 5, 4, 3",
|
||||||
"0, 1, 5, 2, 4, 3",
|
"0, 1, 5, 2, 4, 3, 6, 7",
|
||||||
"0, 5, 1, 2, 4, 3",
|
"0, 5, 6, 7, 1, 2, 4, 3",
|
||||||
"5, 0, 1, 2, 4, 3",
|
"5, 0, 1, 2, 7, 6, 4, 3",
|
||||||
})
|
})
|
||||||
void tieBreaker(int a, int b, int c, int d, int e, int f) {
|
void tieBreaker(int a, int b, int c, int d, int e, int f, int g, int h) {
|
||||||
heap = LongMinHeap.newArrayHeap(6, (id1, id2) -> -Integer.compare(id1, id2));
|
heap = LongMinHeap.newArrayHeap(9, (id1, id2) -> -Integer.compare(id1, id2));
|
||||||
heap.push(a, 0L);
|
heap.push(a, 0L);
|
||||||
heap.push(b, 0L);
|
heap.push(b, 0L);
|
||||||
heap.push(c, 0L);
|
heap.push(c, 0L);
|
||||||
heap.push(d, 0L);
|
heap.push(d, 0L);
|
||||||
heap.push(e, 0L);
|
heap.push(e, 0L);
|
||||||
heap.push(f, 0L);
|
heap.push(f, 0L);
|
||||||
|
heap.push(g, 0L);
|
||||||
|
heap.push(h, 0L);
|
||||||
|
assertEquals(7, heap.poll());
|
||||||
|
assertEquals(6, heap.poll());
|
||||||
assertEquals(5, heap.poll());
|
assertEquals(5, heap.poll());
|
||||||
assertEquals(4, heap.poll());
|
assertEquals(4, heap.poll());
|
||||||
assertEquals(3, heap.poll());
|
assertEquals(3, heap.poll());
|
||||||
|
|
Ładowanie…
Reference in New Issue