diff --git a/src/main/java/com/onthegomap/flatmap/FileUtils.java b/src/main/java/com/onthegomap/flatmap/FileUtils.java index edc5b764..464bd03c 100644 --- a/src/main/java/com/onthegomap/flatmap/FileUtils.java +++ b/src/main/java/com/onthegomap/flatmap/FileUtils.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -26,4 +27,53 @@ public class FileUtils { public static boolean hasExtension(Path path, String extension) { return path.toString().toLowerCase().endsWith("." + extension.toLowerCase()); } + + public static long fileSize(Path path) { + try { + return Files.size(path); + } catch (IOException e) { + return 0; + } + } + + public static long directorySize(Path path) { + try { + return Files.walk(path) + .filter(Files::isRegularFile) + .mapToLong(FileUtils::fileSize) + .sum(); + } catch (IOException e) { + return 0; + } + } + + public static long size(Path path) { + return Files.isDirectory(path) ? directorySize(path) : fileSize(path); + } + + public static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new IllegalStateException("Unable to delete " + path, e); + } + } + + public static void deleteDirectory(Path path) { + try { + Files.walk(path) + .sorted(Comparator.reverseOrder()) + .forEach(FileUtils::deleteFile); + } catch (IOException e) { + throw new IllegalStateException("Unable to delete " + path, e); + } + } + + public static void delete(Path path) { + if (Files.isDirectory(path)) { + deleteDirectory(path); + } else { + deleteFile(path); + } + } } diff --git a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java index 007a68cc..eb888e61 100644 --- a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java +++ b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,9 +59,9 @@ public class OpenMapTilesMain { var translations = Translations.defaultProvider(languages); var profile = new OpenMapTilesProfile(); - FileUtils.forceMkdir(tmpDir.toFile()); + Files.createDirectories(tmpDir); Path nodeDb = tmpDir.resolve("node.db"); - LongLongMap nodeLocations = new LongLongMap.MapdbSortedTable(nodeDb); + LongLongMap nodeLocations = LongLongMap.newFileBackedSortedTable(nodeDb); FeatureSort featureDb = FeatureSort.newExternalMergeSort(tmpDir.resolve("feature.db"), config.threads(), stats); FeatureGroup featureMap = new FeatureGroup(featureDb, profile); FeatureRenderer renderer = new FeatureRenderer(config); diff --git a/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java b/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java index e03b943d..db048052 100644 --- a/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java +++ b/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java @@ -1,5 +1,6 @@ package com.onthegomap.flatmap.collections; +import com.onthegomap.flatmap.FileUtils; import com.onthegomap.flatmap.monitoring.ProcessInfo; import com.onthegomap.flatmap.monitoring.ProgressLoggers; import com.onthegomap.flatmap.monitoring.Stats; @@ -22,7 +23,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; -import org.apache.commons.io.FileUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +67,7 @@ class ExternalMergeSort implements FeatureSort { this.workers = workers; LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers); try { - FileUtils.deleteDirectory(dir.toFile()); + FileUtils.deleteDirectory(dir); Files.createDirectories(dir); newChunk(); } catch (IOException e) { @@ -91,7 +91,7 @@ class ExternalMergeSort implements FeatureSort { @Override public long getStorageSize() { - return FileUtils.sizeOfDirectory(dir.toFile()); + return FileUtils.directorySize(dir); } private static T time(AtomicLong timer, Supplier func) { diff --git a/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java b/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java index fd7b908c..bff92001 100644 --- a/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java +++ b/src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java @@ -1,8 +1,16 @@ package com.onthegomap.flatmap.collections; +import com.onthegomap.flatmap.FileUtils; import java.io.Closeable; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.function.LongSupplier; +import org.mapdb.Serializer; +import org.mapdb.SortedTableMap; +import org.mapdb.volume.ByteArrayVol; +import org.mapdb.volume.MappedFileVol; +import org.mapdb.volume.Volume; public interface LongLongMap extends Closeable { @@ -12,30 +20,82 @@ public interface LongLongMap extends Closeable { long fileSize(); + default long[] multiGet(long[] key) { + long[] result = new long[key.length]; + for (int i = 0; i < key.length; i++) { + result[i] = get(key[i]); + } + return result; + } + + private static Volume prepare(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new IllegalStateException("Unable to delete " + path, e); + } + path.toFile().deleteOnExit(); + return MappedFileVol.FACTORY.makeVolume(path.toAbsolutePath().toString(), false); + } + + private static Volume createInMemoryVolume() { + return ByteArrayVol.FACTORY.makeVolume("", false); + } + + static LongLongMap newFileBackedSortedTable(Path path) { + Volume volume = prepare(path); + return new MapdbSortedTable(volume, () -> FileUtils.size(path)); + } + + static LongLongMap newInMemorySortedTable() { + Volume volume = createInMemoryVolume(); + return new MapdbSortedTable(volume, () -> 0); + } + class MapdbSortedTable implements LongLongMap { - public MapdbSortedTable(Path nodeDb) { + private final SortedTableMap.Sink mapSink; + private volatile SortedTableMap map = null; + private final LongSupplier fileSize; + private MapdbSortedTable(Volume volume, LongSupplier fileSize) { + mapSink = SortedTableMap.create(volume, Serializer.LONG, Serializer.LONG).createFromSink(); + this.fileSize = fileSize; + } + + private SortedTableMap getMap() { + SortedTableMap result = map; + if (result == null) { + synchronized (this) { + result = map; + if (result == null) { + map = mapSink.create(); + } + } + } + return map; } @Override public void put(long key, long value) { - - } - - @Override - public long get(long key) { - return 0; + mapSink.put(key, value); } @Override public long fileSize() { - return 0; + return fileSize.getAsLong(); } @Override - public void close() throws IOException { + public long get(long key) { + return getMap().getOrDefault(key, Long.MIN_VALUE); + } + @Override + public void close() { + if (map != null) { + map.close(); + } } } } diff --git a/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java b/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java new file mode 100644 index 00000000..f67502d0 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java @@ -0,0 +1,67 @@ +package com.onthegomap.flatmap.collections; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public abstract class LongLongMapTest { + + protected LongLongMap map; + + @Test + public void missingValue() { + assertEquals(Long.MIN_VALUE, map.get(0)); + } + + @Test + public void insertLookup() { + map.put(1, 1); + assertEquals(Long.MIN_VALUE, map.get(0)); + assertEquals(1, map.get(1)); + } + + @Test + public void insertMultiLookup() { + map.put(1, 3); + map.put(2, 4); + map.put(Long.MAX_VALUE, Long.MAX_VALUE); + assertEquals(Long.MIN_VALUE, map.get(0)); + assertArrayEquals(new long[]{3, 4, Long.MAX_VALUE, Long.MIN_VALUE}, + map.multiGet(new long[]{1, 2, Long.MAX_VALUE, 3})); + } + + @Test + public void bigMultiInsert() { + long[] key = new long[5000]; + long[] expected = new long[5000]; + for (int i = 0; i < 5000; i++) { + map.put(i, i + 1); + key[i] = i; + expected[i] = i + 1; + } + + long[] result = map.multiGet(key); + + assertArrayEquals(expected, result); + } + + public static class SortedTableFileTest extends LongLongMapTest { + + @BeforeEach + public void setup(@TempDir Path dir) { + this.map = LongLongMap.newFileBackedSortedTable(dir.resolve("test-node-db-sorted")); + } + } + + public static class SortedTableMemoryTest extends LongLongMapTest { + + @BeforeEach + public void setup() { + this.map = LongLongMap.newInMemorySortedTable(); + } + } +}