implemented long long map

src/main/java/com/onthegomap/flatmap/FileUtils.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Comparator;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -26,4 +27,53 @@ public class FileUtils {
   public static boolean hasExtension(Path path, String extension) {
     return path.toString().toLowerCase().endsWith("." + extension.toLowerCase());
   }
+
+  public static long fileSize(Path path) {
+    try {
+      return Files.size(path);
+    } catch (IOException e) {
+      return 0;
+    }
+  }
+
+  public static long directorySize(Path path) {
+    try {
+      return Files.walk(path)
+        .filter(Files::isRegularFile)
+        .mapToLong(FileUtils::fileSize)
+        .sum();
+    } catch (IOException e) {
+      return 0;
+    }
+  }
+
+  public static long size(Path path) {
+    return Files.isDirectory(path) ? directorySize(path) : fileSize(path);
+  }
+
+  public static void deleteFile(Path path) {
+    try {
+      Files.deleteIfExists(path);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to delete " + path, e);
+    }
+  }
+
+  public static void deleteDirectory(Path path) {
+    try {
+      Files.walk(path)
+        .sorted(Comparator.reverseOrder())
+        .forEach(FileUtils::deleteFile);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to delete " + path, e);
+    }
+  }
+
+  public static void delete(Path path) {
+    if (Files.isDirectory(path)) {
+      deleteDirectory(path);
+    } else {
+      deleteFile(path);
+    }
+  }
 }

src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java

@@ -13,7 +13,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
-import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -60,9 +59,9 @@ public class OpenMapTilesMain {
     var translations = Translations.defaultProvider(languages);
     var profile = new OpenMapTilesProfile();
 
-    FileUtils.forceMkdir(tmpDir.toFile());
+    Files.createDirectories(tmpDir);
     Path nodeDb = tmpDir.resolve("node.db");
-    LongLongMap nodeLocations = new LongLongMap.MapdbSortedTable(nodeDb);
+    LongLongMap nodeLocations = LongLongMap.newFileBackedSortedTable(nodeDb);
     FeatureSort featureDb = FeatureSort.newExternalMergeSort(tmpDir.resolve("feature.db"), config.threads(), stats);
     FeatureGroup featureMap = new FeatureGroup(featureDb, profile);
     FeatureRenderer renderer = new FeatureRenderer(config);

src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java

@@ -1,5 +1,6 @@
 package com.onthegomap.flatmap.collections;
 
+import com.onthegomap.flatmap.FileUtils;
 import com.onthegomap.flatmap.monitoring.ProcessInfo;
 import com.onthegomap.flatmap.monitoring.ProgressLoggers;
 import com.onthegomap.flatmap.monitoring.Stats;
@@ -22,7 +23,6 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
-import org.apache.commons.io.FileUtils;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,7 +67,7 @@ class ExternalMergeSort implements FeatureSort {
     this.workers = workers;
     LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers);
     try {
-      FileUtils.deleteDirectory(dir.toFile());
+      FileUtils.deleteDirectory(dir);
       Files.createDirectories(dir);
       newChunk();
     } catch (IOException e) {
@@ -91,7 +91,7 @@ class ExternalMergeSort implements FeatureSort {
 
   @Override
   public long getStorageSize() {
-    return FileUtils.sizeOfDirectory(dir.toFile());
+    return FileUtils.directorySize(dir);
   }
 
   private static <T> T time(AtomicLong timer, Supplier<T> func) {

src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java

@@ -1,8 +1,16 @@
 package com.onthegomap.flatmap.collections;
 
+import com.onthegomap.flatmap.FileUtils;
 import java.io.Closeable;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.function.LongSupplier;
+import org.mapdb.Serializer;
+import org.mapdb.SortedTableMap;
+import org.mapdb.volume.ByteArrayVol;
+import org.mapdb.volume.MappedFileVol;
+import org.mapdb.volume.Volume;
 
 public interface LongLongMap extends Closeable {
 
@@ -12,30 +20,82 @@ public interface LongLongMap extends Closeable {
 
   long fileSize();
 
+  default long[] multiGet(long[] key) {
+    long[] result = new long[key.length];
+    for (int i = 0; i < key.length; i++) {
+      result[i] = get(key[i]);
+    }
+    return result;
+  }
+
+  private static Volume prepare(Path path) {
+    try {
+      Files.deleteIfExists(path);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to delete " + path, e);
+    }
+    path.toFile().deleteOnExit();
+    return MappedFileVol.FACTORY.makeVolume(path.toAbsolutePath().toString(), false);
+  }
+
+  private static Volume createInMemoryVolume() {
+    return ByteArrayVol.FACTORY.makeVolume("", false);
+  }
+
+  static LongLongMap newFileBackedSortedTable(Path path) {
+    Volume volume = prepare(path);
+    return new MapdbSortedTable(volume, () -> FileUtils.size(path));
+  }
+
+  static LongLongMap newInMemorySortedTable() {
+    Volume volume = createInMemoryVolume();
+    return new MapdbSortedTable(volume, () -> 0);
+  }
+
   class MapdbSortedTable implements LongLongMap {
 
-    public MapdbSortedTable(Path nodeDb) {
+    private final SortedTableMap.Sink<Long, Long> mapSink;
+    private volatile SortedTableMap<Long, Long> map = null;
+    private final LongSupplier fileSize;
 
+    private MapdbSortedTable(Volume volume, LongSupplier fileSize) {
+      mapSink = SortedTableMap.create(volume, Serializer.LONG, Serializer.LONG).createFromSink();
+      this.fileSize = fileSize;
+    }
+
+    private SortedTableMap<Long, Long> getMap() {
+      SortedTableMap<Long, Long> result = map;
+      if (result == null) {
+        synchronized (this) {
+          result = map;
+          if (result == null) {
+            map = mapSink.create();
+          }
+        }
+      }
+      return map;
     }
 
     @Override
     public void put(long key, long value) {
-
-    }
-
-    @Override
-    public long get(long key) {
-      return 0;
+      mapSink.put(key, value);
     }
 
     @Override
     public long fileSize() {
-      return 0;
+      return fileSize.getAsLong();
     }
 
     @Override
-    public void close() throws IOException {
+    public long get(long key) {
+      return getMap().getOrDefault(key, Long.MIN_VALUE);
+    }
 
+    @Override
+    public void close() {
+      if (map != null) {
+        map.close();
+      }
     }
   }
 }

src/test/java/com/onthegomap/flatmap/collections/LongLongMapTest.java

@@ -0,0 +1,67 @@
+package com.onthegomap.flatmap.collections;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.nio.file.Path;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public abstract class LongLongMapTest {
+
+  protected LongLongMap map;
+
+  @Test
+  public void missingValue() {
+    assertEquals(Long.MIN_VALUE, map.get(0));
+  }
+
+  @Test
+  public void insertLookup() {
+    map.put(1, 1);
+    assertEquals(Long.MIN_VALUE, map.get(0));
+    assertEquals(1, map.get(1));
+  }
+
+  @Test
+  public void insertMultiLookup() {
+    map.put(1, 3);
+    map.put(2, 4);
+    map.put(Long.MAX_VALUE, Long.MAX_VALUE);
+    assertEquals(Long.MIN_VALUE, map.get(0));
+    assertArrayEquals(new long[]{3, 4, Long.MAX_VALUE, Long.MIN_VALUE},
+      map.multiGet(new long[]{1, 2, Long.MAX_VALUE, 3}));
+  }
+
+  @Test
+  public void bigMultiInsert() {
+    long[] key = new long[5000];
+    long[] expected = new long[5000];
+    for (int i = 0; i < 5000; i++) {
+      map.put(i, i + 1);
+      key[i] = i;
+      expected[i] = i + 1;
+    }
+
+    long[] result = map.multiGet(key);
+
+    assertArrayEquals(expected, result);
+  }
+
+  public static class SortedTableFileTest extends LongLongMapTest {
+
+    @BeforeEach
+    public void setup(@TempDir Path dir) {
+      this.map = LongLongMap.newFileBackedSortedTable(dir.resolve("test-node-db-sorted"));
+    }
+  }
+
+  public static class SortedTableMemoryTest extends LongLongMapTest {
+
+    @BeforeEach
+    public void setup() {
+      this.map = LongLongMap.newInMemorySortedTable();
+    }
+  }
+}