diff --git a/core/pom.xml b/core/pom.xml index 83ee4451..5fb8acb0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -50,11 +50,6 @@ mapdb 3.0.8 - - org.rocksdb - rocksdbjni - 6.20.3 - org.msgpack msgpack-core diff --git a/core/src/main/java/com/onthegomap/flatmap/FlatMapRunner.java b/core/src/main/java/com/onthegomap/flatmap/FlatMapRunner.java index 7bf0c8a9..7211093e 100644 --- a/core/src/main/java/com/onthegomap/flatmap/FlatMapRunner.java +++ b/core/src/main/java/com/onthegomap/flatmap/FlatMapRunner.java @@ -57,10 +57,11 @@ public class FlatMapRunner { return switch (config.longLongMap()) { case "mapdb" -> LongLongMap.newFileBackedSortedTable(nodeDbPath); case "sparsearray" -> LongLongMap.newFileBackedSparseArray(nodeDbPath); + case "sparsemem2" -> LongLongMap.newInMemorySparseArray2(); + case "sparse2" -> LongLongMap.newFileBackedSparseArray2(nodeDbPath); case "ramsparsearray" -> LongLongMap.newInMemorySparseArray(); case "ramarray" -> LongLongMap.newArrayBacked(); case "sqlite" -> LongLongMap.newSqlite(nodeDbPath); - case "rocksdb" -> LongLongMap.newRocksdb(nodeDbPath); default -> throw new IllegalStateException("Unexpected llmap value: " + config.longLongMap()); }; } diff --git a/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java b/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java index ac05c04f..5cefda96 100644 --- a/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java +++ b/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java @@ -49,6 +49,18 @@ public class MemoryEstimator { return object == null ? 0 : (24L + 8L * object.keys.length + 24L + 4L * object.values.length); } + public static long size(int[] array) { + return 24L + 4L * array.length; + } + + public static long size(long[] array) { + return 24L + 8L * array.length; + } + + public static long size(Object[] array) { + return 24L + 8L * array.length; + } + public interface HasEstimate { long estimateMemoryUsageBytes(); diff --git a/core/src/main/java/com/onthegomap/flatmap/Wikidata.java b/core/src/main/java/com/onthegomap/flatmap/Wikidata.java index 2fc2c3fd..c789c490 100644 --- a/core/src/main/java/com/onthegomap/flatmap/Wikidata.java +++ b/core/src/main/java/com/onthegomap/flatmap/Wikidata.java @@ -133,8 +133,9 @@ public class Wikidata { Wikidata fetcher = new Wikidata(writer, Client.wrap(client), 5_000, profile); fetcher.loadExisting(oldMappings); + String pbfParsePrefix = "pbfwikidata"; var topology = Topology.start("wikidata", stats) - .fromGenerator("pbf", infile.read("pbfwikidata", readerThreads)) + .fromGenerator("pbf", infile.read(pbfParsePrefix, readerThreads)) .addBuffer("reader_queue", 50_000, 10_000) .addWorker("filter", processThreads, fetcher::filter) .addBuffer("fetch_queue", 50_000) @@ -153,7 +154,7 @@ public class Wikidata { .addRateCounter("wiki", fetcher.wikidatas) .addFileSize(outfile) .addProcessStats() - .addThreadPoolStats("parse", "pool-") + .addThreadPoolStats("parse", pbfParsePrefix) .addTopologyStats(topology); topology.awaitAndLog(loggers, config.logInterval()); diff --git a/core/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java b/core/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java index 4900e263..d23344dc 100644 --- a/core/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java +++ b/core/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java @@ -5,7 +5,6 @@ import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND; import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongLongHashMap; import com.graphhopper.coll.GHLongLongHashMap; -import com.graphhopper.util.StopWatch; import com.onthegomap.flatmap.FileUtils; import com.onthegomap.flatmap.Format; import com.onthegomap.flatmap.MemoryEstimator; @@ -43,12 +42,6 @@ import org.mapdb.SortedTableMap; import org.mapdb.volume.ByteArrayVol; import org.mapdb.volume.MappedFileVol; import org.mapdb.volume.Volume; -import org.rocksdb.EnvOptions; -import org.rocksdb.IngestExternalFileOptions; -import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.SstFileWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sqlite.SQLiteConfig; @@ -59,11 +52,12 @@ public interface LongLongMap extends Closeable { Path path = Path.of("./llmaptest"); FileUtils.delete(path); LongLongMap map = switch (args[0]) { + case "sparsemem2" -> new SparseArray2Memory(); case "sparsearraymemory" -> new SparseArrayMemory(); case "hppc" -> new HppcMap(); case "array" -> new Array(); - case "rocksdb" -> newRocksdb(path); + case "sparse2" -> new SparseArray2(path); case "sqlite" -> newSqlite(path); case "sparsearray" -> new SparseArray(path); case "mapdb" -> newFileBackedSortedTable(path); @@ -71,7 +65,6 @@ public interface LongLongMap extends Closeable { }; long entries = Long.parseLong(args[1]); int readers = Integer.parseInt(args[2]); - int batchSize = Integer.parseInt(args[3]); class LocalCounter { @@ -84,8 +77,8 @@ public interface LongLongMap extends Closeable { AtomicReference writeRate = new AtomicReference<>(); new Worker("writer", new Stats.InMemory(), 1, () -> { long start = System.nanoTime(); - for (long i = 1; i <= entries; i++) { - map.put(i, i + 1); + for (long i = 0; i < entries; i++) { + map.put(i + 1L, i + 2L); counter.count = i; } long end = System.nanoTime(); @@ -111,15 +104,21 @@ public interface LongLongMap extends Closeable { } catch (InterruptedException e) { throw new RuntimeException(e); } - LongArrayList keys = new LongArrayList(batchSize); Random random = new Random(rnum); - while (true) { - keys.elementsCount = 0; - for (int j = 0; j < batchSize; j++) { - keys.add((Math.abs(random.nextLong()) % entries) + 1); + try { + long sum = 0; + long b = 0; + while (b == 0) { + readCount.inc(); + long key = 1L + (Math.abs(random.nextLong()) % entries); + long value = map.get(key); + assert key + 1 == value : key + " value was " + value; + sum += value; } - readCount.incBy(batchSize); - map.multiGet(keys); + System.err.println(sum); + } catch (Throwable e) { + e.printStackTrace(); + System.exit(1); } }).start(); } @@ -152,8 +151,12 @@ public interface LongLongMap extends Closeable { long MISSING_VALUE = Long.MIN_VALUE; - static LongLongMap newRocksdb(Path path) { - return new RocksdbLongLongMap(path); + static LongLongMap newInMemorySparseArray2() { + return new SparseArray2Memory(); + } + + static LongLongMap newFileBackedSparseArray2(Path nodeDbPath) { + return new SparseArray2(nodeDbPath); } void put(long key, long value); @@ -304,40 +307,42 @@ public interface LongLongMap extends Closeable { class Array implements LongLongMap { int used = 0; - private static final long MAX_MEM_USAGE = 100_000_000_000L; // 100GB - private static final long INDEX_OVERHEAD = 256_000_000; // 256mb - private static final long MAX_ENTRIES = MAX_MEM_USAGE / 8L; - private static final long MAX_SEGMENTS = INDEX_OVERHEAD / (24 + 8); - private static final long SEGMENT_SIZE = MAX_ENTRIES / MAX_SEGMENTS + 1; + private static final long MAX_ENTRIES = 10_000_000_000L; // 10B + private static final long OFFSET_BITS = 10; + private static final long SEGMENT_SIZE = 1 << OFFSET_BITS; // 1024 + private static final long OFFSET_MASK = SEGMENT_SIZE - 1L; + private static final long MAX_SEGMENTS = MAX_ENTRIES / SEGMENT_SIZE + 1; private long[][] longs = new long[(int) MAX_SEGMENTS][]; @Override public void put(long key, long value) { - int segment = (int) (key / SEGMENT_SIZE); + int segment = (int) (key >>> OFFSET_BITS); long[] seg = longs[segment]; if (seg == null) { - seg = longs[segment] = new long[(int) SEGMENT_SIZE]; + longs[segment] = seg = new long[(int) SEGMENT_SIZE]; Arrays.fill(seg, MISSING_VALUE); used++; } - seg[(int) (key % SEGMENT_SIZE)] = value; + seg[(int) (key & OFFSET_MASK)] = value; } @Override public long get(long key) { - long[] segment = longs[(int) (key / SEGMENT_SIZE)]; - return segment == null ? MISSING_VALUE : segment[(int) (key % SEGMENT_SIZE)]; + long[] segment = longs[(int) (key >>> OFFSET_BITS)]; + return segment == null ? MISSING_VALUE : segment[(int) (key & OFFSET_MASK)]; } @Override public long fileSize() { - return 24L + 8L * longs.length + ((long) used) * (24L + 8L * SEGMENT_SIZE); + return MemoryEstimator.size(longs) + ((long) used) * (24L + 8L * SEGMENT_SIZE); } @Override public void close() throws IOException { - Arrays.fill(longs, null); + if (longs != null) { + Arrays.fill(longs, null); + } longs = null; } } @@ -634,7 +639,6 @@ public interface LongLongMap extends Closeable { synchronized (this) { if (!readable) { try { - System.err.println("Making readable"); flush(); conn.close(); } catch (SQLException e) { @@ -823,123 +827,183 @@ public interface LongLongMap extends Closeable { | ((long) b[7] & 0xff); } - class RocksdbLongLongMap implements LongLongMap { + class SparseArray2 implements LongLongMap { - private final Path sst; - private final SstFileWriter writer; + private static final long MAX_KEY = 10_000_000_000L; + private static final long OFFSET_BITS = 7; // 128 + private static final long SLAB_BITS = 17; // 1MB of longs + private static final int OFFSET_MASK = (1 << OFFSET_BITS) - 1; + private static final long SLAB_MINUS_MASK = SLAB_BITS - OFFSET_BITS; + private static final int SLAB_PRE_MASK = (1 << SLAB_MINUS_MASK) - 1; + private int[] keyIndex = new int[(int) (MAX_KEY >>> OFFSET_BITS)]; // 300mb + private int lastChunk = -1; + private int slab = -1; + private final Path path; + private final DataOutputStream outputStream; + private int lastOffset = 0; + private FileChannel channel; + private long outIdx = 0; - static { - RocksDB.loadLibrary(); - } - - private Path path = null; - - public RocksdbLongLongMap(Path path) { + SparseArray2(Path path) { this.path = path; try { - FileUtils.delete(path); - Files.createDirectories(path); - writer = new SstFileWriter(new EnvOptions(), new Options()); - FileUtils.delete(Path.of("/tmp/rocks")); - Files.createDirectories(Path.of("/tmp/rocks")); - sst = Path.of("/tmp/rocks").resolve("sst"); - writer.open(sst.toAbsolutePath().toString()); - } catch (RocksDBException | IOException ex) { - throw new RuntimeException(ex); + this.outputStream = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(path), 50_000)); + } catch (IOException e) { + throw new IllegalStateException("Could not create compact array output stream", e); } + Arrays.fill(keyIndex, -1); } - private volatile boolean compacted = false; - - private ThreadLocal rocksDb = ThreadLocal.withInitial(() -> { - try { - return RocksDB.openReadOnly(new Options() - .setAllowMmapReads(true) - , path.resolve("db").toAbsolutePath().toString()); - } catch (RocksDBException e) { - return null; - } - }); - - private synchronized void compactIfNecessary() { - if (!compacted) { - synchronized (this) { - if (!compacted) { - try { - System.err.println("Ingesting..."); - StopWatch watch = new StopWatch().start(); - writer.finish(); - writer.close(); - try (RocksDB _db = RocksDB.open(new Options().setCreateIfMissing(true), - path.resolve("db").toAbsolutePath().toString())) { - _db.ingestExternalFile(List.of(sst.toAbsolutePath().toString()), new IngestExternalFileOptions()); - } finally { - FileUtils.delete(sst); - } - - System.err.println("Done. Took " + (watch.stop().getCurrentSeconds()) + "s"); - } catch (RocksDBException e) { - throw new Error(e); - } - compacted = true; - } - } - } + private void appendValue(long value) throws IOException { + outputStream.writeLong(value); + lastOffset = (lastOffset + 1) & OFFSET_MASK; + outIdx++; } @Override - public void put(long key, long val) { + public void put(long key, long value) { try { - writer.put(LongLongMap.encodeNodeId(key), LongLongMap.encodeNodeValue(val)); - } catch (RocksDBException e) { - throw new RuntimeException(e); + int chunk = (int) (key >>> OFFSET_BITS); + int offset = (int) (key & OFFSET_MASK); + if (lastChunk != chunk) { + keyIndex[chunk] = ++slab; + lastChunk = chunk; + while (lastOffset != 0) { + appendValue(MISSING_VALUE); + } + } + while (lastOffset != offset) { + appendValue(MISSING_VALUE); + } + appendValue(value); + } catch (IOException e) { + throw new IllegalStateException(e); } } + private volatile MappedByteBuffer[] segments; + @Override public long get(long key) { - compactIfNecessary(); - try { - byte[] results = rocksDb.get().get(LongLongMap.encodeNodeId(key)); - return results == null ? MISSING_VALUE : LongLongMap.decodeNodeValue(results); - } catch (RocksDBException e) { - throw new RuntimeException(e); + if (segments == null) { + synchronized (this) { + if (segments == null) { + build(); + } + } } + int chunk = (int) (key >>> OFFSET_BITS); + int offset = (int) (key & OFFSET_MASK); + int slab = keyIndex[chunk]; + if (slab == -1) { + return MISSING_VALUE; + } + MappedByteBuffer segment = segments[slab >>> SLAB_MINUS_MASK]; + int idx = (((slab & SLAB_PRE_MASK) << OFFSET_BITS) | offset) << 3; + return idx >= segment.limit() ? MISSING_VALUE : segment.getLong(idx); } - @Override - public long[] multiGet(LongArrayList key) { - compactIfNecessary(); - long[] result = new long[key.size()]; + private void build() { try { - List keys = new ArrayList<>(key.size()); - for (int i = 0; i < key.size(); i++) { - keys.add(LongLongMap.encodeNodeId(key.get(i))); + outputStream.close(); + channel = FileChannel.open(path, StandardOpenOption.READ); + var segmentCount = (slab >>> SLAB_MINUS_MASK) + 1; + MappedByteBuffer[] result = new MappedByteBuffer[segmentCount]; + for (int i = 0; i < segmentCount; i++) { + long start = ((long) i) << SLAB_BITS; + if (outIdx > start) { + result[i] = channel + .map(FileChannel.MapMode.READ_ONLY, start << 3, Math.min(1 << SLAB_BITS, outIdx - start) << 3); + } } - List results = rocksDb.get().multiGetAsList(keys); - for (int i = 0; i < results.size(); i++) { - byte[] thisResult = results.get(i); - result[i] = thisResult == null ? MISSING_VALUE : LongLongMap.decodeNodeValue(thisResult); - } - return result; - } catch (RocksDBException e) { - throw new RuntimeException(e); + segments = result; + } catch (IOException e) { + throw new IllegalStateException("Could not create segments", e); } } - @Override - public long[] multiGet(long[] key) { - return multiGet(LongArrayList.from(key)); - } - @Override public long fileSize() { return FileUtils.size(path); } @Override - public void close() { -// db.close(); + public void close() throws IOException { + if (channel != null) { + channel.close(); + channel = null; + } + if (segments != null) { + Arrays.fill(segments, null); + segments = null; + } + keyIndex = null; + } + } + + class SparseArray2Memory implements LongLongMap { + + private static final long MAX_KEY = 10_000_000_000L; + private static final long OFFSET_BITS = 8; // 256 + private static final long SLAB_BITS = 16; // 8MB of longs + private static final long OFFSET_MASK = (1L << OFFSET_BITS) - 1; + private static final long SLAB_MINUS_MASK = SLAB_BITS - OFFSET_BITS; + private static final int SLAB_PRE_MASK = (1 << SLAB_MINUS_MASK) - 1; + private final int[] keyIndex = new int[(int) (MAX_KEY >>> OFFSET_BITS)]; // 300mb + private final long[][] valueSlabs = new long[(int) (MAX_KEY >>> SLAB_BITS)][]; + private int lastChunk = -1; + private int slab = -1; + private long slabSizes = 0; + + SparseArray2Memory() { + Arrays.fill(keyIndex, -1); + } + + @Override + public void put(long key, long value) { + int chunk = (int) (key >>> OFFSET_BITS); + int offset = (int) (key & OFFSET_MASK); + if (lastChunk != chunk) { + slab++; + keyIndex[chunk] = slab; + lastChunk = chunk; + } + int slabNum = slab >>> SLAB_MINUS_MASK; + long[] theSlab = valueSlabs[slabNum]; + if (theSlab == null) { + theSlab = new long[1 << SLAB_BITS]; + valueSlabs[slabNum] = theSlab; + for (int i = 0, len = theSlab.length; i < len; i++) { + theSlab[i] = MISSING_VALUE; + } + slabSizes += 24L + 8L * theSlab.length; + } + theSlab[((slab & SLAB_PRE_MASK) << OFFSET_BITS) | offset] = value; + } + + @Override + public long get(long key) { + int chunk = (int) (key >>> OFFSET_BITS); + int offset = (int) (key & OFFSET_MASK); + int slab = keyIndex[chunk]; + if (slab == -1) { + return MISSING_VALUE; + } + long[] theSlab = valueSlabs[slab >>> SLAB_MINUS_MASK]; + return theSlab[((slab & SLAB_PRE_MASK) << OFFSET_BITS) | offset]; + } + + @Override + public long fileSize() { + return 24L + 4L * keyIndex.length + + 24L + 8L * valueSlabs.length + + slabSizes; + } + + @Override + public void close() throws IOException { +// valueSlabs = null; +// keyIndex = null; } } } diff --git a/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java b/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java index 8a2571af..783f204d 100644 --- a/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java +++ b/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java @@ -96,7 +96,8 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima public void pass1(CommonParams config) { var timer = stats.startTimer(name + "_pass1"); - var topology = Topology.start("osm_pass1", stats) + String pbfParsePrefix = "pbfpass1"; + var topology = Topology.start(pbfParsePrefix, stats) .fromGenerator("pbf", osmInputFile.read("pbfpass1", config.threads() - 1)) .addBuffer("reader_queue", 50_000, 10_000) .sinkToConsumer("process", 1, this::processPass1); @@ -108,7 +109,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima .addRateCounter("rels", PASS1_RELATIONS) .addProcessStats() .addInMemoryObject("hppc", this) - .addThreadPoolStats("parse", "pool-") + .addThreadPoolStats("parse", pbfParsePrefix) .addTopologyStats(topology); topology.awaitAndLog(loggers, config.logInterval()); timer.stop(); @@ -152,7 +153,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima public void pass2(FeatureGroup writer, CommonParams config) { var timer = stats.startTimer(name + "_pass2"); int readerThreads = Math.max(config.threads() / 4, 1); - int processThreads = config.threads(); + int processThreads = config.threads() - 1; Counter.MultiThreadCounter nodesProcessed = Counter.newMultiThreadCounter(); Counter.MultiThreadCounter waysProcessed = Counter.newMultiThreadCounter(); Counter.MultiThreadCounter relsProcessed = Counter.newMultiThreadCounter(); @@ -164,8 +165,9 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima CountDownLatch waysDone = new CountDownLatch(processThreads); + String parseThreadPrefix = "pbfpass2"; var topology = Topology.start("osm_pass2", stats) - .fromGenerator("pbf", osmInputFile.read("pbfpass2", readerThreads)) + .fromGenerator("pbf", osmInputFile.read(parseThreadPrefix, readerThreads)) .addBuffer("reader_queue", 50_000, 1_000) .addWorker("process", processThreads, (prev, next) -> { Counter nodes = nodesProcessed.counterForThread(); @@ -217,7 +219,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima .addFileSize(writer::getStorageSize) .addProcessStats() .addInMemoryObject("hppc", this) - .addThreadPoolStats("parse", "pool-") + .addThreadPoolStats("parse", parseThreadPrefix) .addTopologyStats(topology); topology.awaitAndLog(logger, config.logInterval()); @@ -545,9 +547,8 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima int num = nodeIds.size(); CoordinateSequence seq = new PackedCoordinateSequence.Double(nodeIds.size(), 2, 0); - long[] values = nodeDb.multiGet(nodeIds); for (int i = 0; i < num; i++) { - long encoded = values[i]; + long encoded = nodeDb.get(nodeIds.get(i)); if (encoded == LongLongMap.MISSING_VALUE) { throw new IllegalArgumentException("Missing location for node: " + nodeIds.get(i)); } diff --git a/core/src/main/java/com/onthegomap/flatmap/worker/Topology.java b/core/src/main/java/com/onthegomap/flatmap/worker/Topology.java index 77260706..e3b65409 100644 --- a/core/src/main/java/com/onthegomap/flatmap/worker/Topology.java +++ b/core/src/main/java/com/onthegomap/flatmap/worker/Topology.java @@ -110,7 +110,7 @@ public record Topology( } public Builder readFromQueue(WorkQueue input) { - return new Builder<>(input, stats); + return new Builder<>(prefix, input, stats); } } @@ -127,8 +127,8 @@ public record Topology( this(prefix, name, null, null, outputQueue, worker, stats); } - public Builder(WorkQueue outputQueue, Stats stats) { - this(null, null, outputQueue, null, stats); + public Builder(String prefix, WorkQueue outputQueue, Stats stats) { + this(prefix, null, outputQueue, null, stats); } public Bufferable addWorker(String name, int threads, WorkerStep step) { diff --git a/core/src/main/resources/assemblies/fatjar.xml b/core/src/main/resources/assemblies/fatjar.xml new file mode 100644 index 00000000..c94f3adc --- /dev/null +++ b/core/src/main/resources/assemblies/fatjar.xml @@ -0,0 +1,24 @@ + + + + fatjar + + jar + + false + + + metaInf-services + + + + + / + true + true + runtime + + + diff --git a/core/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java b/core/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java index b9719839..e6709808 100644 --- a/core/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java +++ b/core/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java @@ -59,11 +59,11 @@ public abstract class LongLongMapTest { @Test public void bigMultiInsert() { - long[] key = new long[5000]; - long[] expected = new long[5000]; - for (int i = 0; i < 5000; i++) { - map.put(i, i + 1); - key[i] = i; + long[] key = new long[50000]; + long[] expected = new long[50000]; + for (int i = 0; i < 50000; i++) { + map.put(i * 4, i + 1); + key[i] = i * 4; expected[i] = i + 1; } @@ -120,11 +120,19 @@ public abstract class LongLongMapTest { } } - public static class RocksdbTest extends LongLongMapTest { + public static class SpareArray2Memory extends LongLongMapTest { + + @BeforeEach + public void setup() { + this.map = new LongLongMap.SparseArray2Memory(); + } + } + + public static class SparseArray2 extends LongLongMapTest { @BeforeEach public void setup(@TempDir Path dir) { - this.map = LongLongMap.newRocksdb(dir.resolve("rocksdb-long-long-map-test")); + this.map = new LongLongMap.SparseArray2(dir.resolve("temp-sparse-array-2")); } } } diff --git a/examples/pom.xml b/examples/pom.xml index 6d430bb9..19c96a05 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -26,10 +26,18 @@ org.apache.maven.plugins maven-assembly-plugin 3.3.0 + + + + com.onthegomap + flatmap-core + ${project.parent.version} + + - jar-with-dependencies + fatjar diff --git a/openmaptiles/pom.xml b/openmaptiles/pom.xml index c7a0e7bf..38737dd2 100644 --- a/openmaptiles/pom.xml +++ b/openmaptiles/pom.xml @@ -49,6 +49,14 @@ org.apache.maven.plugins maven-assembly-plugin 3.3.0 + + + + com.onthegomap + flatmap-core + ${project.parent.version} + + @@ -57,7 +65,7 @@ - jar-with-dependencies + fatjar diff --git a/quickstart.sh b/quickstart.sh index e12d9115..f50c16b0 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -4,7 +4,7 @@ set -o errexit set -o pipefail set -o nounset -JAR="openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-jar-with-dependencies.jar" +JAR="openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-fatjar.jar" echo "Downloading data..." AREA="${1:-north-america_us_massachusetts}" diff --git a/scripts/ubuntu_vm_setup.sh b/scripts/ubuntu_vm_setup.sh index 3a8e0983..bca653aa 100755 --- a/scripts/ubuntu_vm_setup.sh +++ b/scripts/ubuntu_vm_setup.sh @@ -12,7 +12,7 @@ fi "$(dirname "$0")"/build.sh -rsync -avzP openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-jar-with-dependencies.jar "${1}":flatmap.jar +rsync -avzP openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-fatjar.jar "${1}":flatmap.jar scp scripts/download-other-sources.sh "${1}":download-other-sources.sh scp scripts/download-osm.sh "${1}":download-osm.sh ssh "${1}" "bash -s" <