simplify sqlite connection config

src/main/java/com/onthegomap/flatmap/write/Mbtiles.java

@@ -37,6 +37,9 @@ import org.sqlite.SQLiteConfig;
 
 public final class Mbtiles implements Closeable {
 
+  // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
+  private static final int MBTILES_APPLICATION_ID = 0x4d504258;
+
   public static final String TILES_TABLE = "tiles";
   public static final String TILES_COL_X = "tile_column";
   public static final String TILES_COL_Y = "tile_row";
@@ -67,16 +70,24 @@ public final class Mbtiles implements Closeable {
 
   public static Mbtiles newInMemoryDatabase() {
     try {
-      return new Mbtiles(DriverManager.getConnection("jdbc:sqlite::memory:")).init();
-
+      SQLiteConfig config = new SQLiteConfig();
+      config.setApplicationId(MBTILES_APPLICATION_ID);
+      return new Mbtiles(DriverManager.getConnection("jdbc:sqlite::memory:", config.toProperties()));
     } catch (SQLException throwables) {
       throw new IllegalStateException("Unable to create in-memory database", throwables);
     }
   }
 
-  public static Mbtiles newFileDatabase(Path path) {
+  public static Mbtiles newWriteToFileDatabase(Path path) {
     try {
-      return new Mbtiles(DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath())).init();
+      SQLiteConfig config = new SQLiteConfig();
+      config.setSynchronous(SQLiteConfig.SynchronousMode.OFF);
+      config.setJournalMode(SQLiteConfig.JournalMode.OFF);
+      config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
+      config.setApplicationId(MBTILES_APPLICATION_ID);
+      config.setPageSize(8_192);
+      config.setPragma(SQLiteConfig.Pragma.MMAP_SIZE, "30000000000");
+      return new Mbtiles(DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties()));
     } catch (SQLException throwables) {
       throw new IllegalArgumentException("Unable to open " + path, throwables);
     }
@@ -86,6 +97,11 @@ public final class Mbtiles implements Closeable {
     try {
       SQLiteConfig config = new SQLiteConfig();
       config.setReadOnly(true);
+      config.setCacheSize(100_000);
+      config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
+      config.setPageSize(32_768);
+      // helps with 3 or more threads concurrently accessing:
+      // config.setOpenMode(SQLiteOpenMode.NOMUTEX);
       Connection connection = DriverManager
         .getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties());
       return new Mbtiles(connection);
@@ -103,15 +119,6 @@ public final class Mbtiles implements Closeable {
     }
   }
 
-  private String pragma(SQLiteConfig.Pragma pragma, Object value) {
-    return "PRAGMA " + pragma.pragmaName + " = " + value + ";";
-  }
-
-  private Mbtiles init() {
-    // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
-    return execute(pragma(SQLiteConfig.Pragma.APPLICATION_ID, "0x4d504258"));
-  }
-
   private Mbtiles execute(String... queries) {
     for (String query : queries) {
       try (var statement = connection.createStatement()) {
@@ -142,16 +149,6 @@ public final class Mbtiles implements Closeable {
     );
   }
 
-  public Mbtiles tuneForWrites() {
-    return execute(
-      pragma(SQLiteConfig.Pragma.SYNCHRONOUS, SQLiteConfig.SynchronousMode.OFF),
-      pragma(SQLiteConfig.Pragma.JOURNAL_MODE, SQLiteConfig.JournalMode.OFF),
-      pragma(SQLiteConfig.Pragma.LOCKING_MODE, SQLiteConfig.LockingMode.EXCLUSIVE),
-      pragma(SQLiteConfig.Pragma.PAGE_SIZE, 8192),
-      pragma(SQLiteConfig.Pragma.MMAP_SIZE, 30000000000L)
-    );
-  }
-
   public Mbtiles vacuumAnalyze() {
     return execute(
       "VACUUM;",
@@ -167,24 +164,27 @@ public final class Mbtiles implements Closeable {
     return new Metadata();
   }
 
-  private PreparedStatement newGetTileStatement() {
-    try {
-      return connection.prepareStatement("""
-        SELECT tile_data FROM %s
-        WHERE tile_column=?
-        AND tile_row=?
-        AND zoom_level=?
-        """.formatted(TILES_TABLE));
-    } catch (SQLException throwables) {
-      throw new IllegalStateException(throwables);
-    }
-  }
+  private PreparedStatement getTileStatement = null;
 
-  private ThreadLocal<PreparedStatement> getTileStatementCache = ThreadLocal.withInitial(this::newGetTileStatement);
+  private PreparedStatement getTileStatement() {
+    if (getTileStatement == null) {
+      try {
+        getTileStatement = connection.prepareStatement("""
+          SELECT tile_data FROM %s
+          WHERE tile_column=?
+          AND tile_row=?
+          AND zoom_level=?
+          """.formatted(TILES_TABLE));
+      } catch (SQLException throwables) {
+        throw new IllegalStateException(throwables);
+      }
+    }
+    return getTileStatement;
+  }
 
   public byte[] getTile(int x, int y, int z) {
     try {
-      PreparedStatement stmt = getTileStatementCache.get();
+      PreparedStatement stmt = getTileStatement();
       stmt.setInt(1, x);
       stmt.setInt(2, (1 << z) - 1 - y);
       stmt.setInt(3, z);
@@ -196,6 +196,22 @@ public final class Mbtiles implements Closeable {
     }
   }
 
+  public List<TileCoord> getAllTileCoords() {
+    List<TileCoord> result = new ArrayList<>();
+    try (Statement statement = connection.createStatement()) {
+      ResultSet rs = statement.executeQuery("select zoom_level, tile_column, tile_row, tile_data from tiles");
+      while (rs.next()) {
+        int z = rs.getInt("zoom_level");
+        int rawy = rs.getInt("tile_row");
+        int x = rs.getInt("tile_column");
+        result.add(TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z));
+      }
+    } catch (SQLException throwables) {
+      throw new IllegalStateException("Could not get all tile coordinates", throwables);
+    }
+    return result;
+  }
+
   public Connection connection() {
     return connection;
   }

src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java

@@ -43,7 +43,7 @@ public class MbtilesWriter {
 
   public static void writeOutput(FeatureGroup features, Path outputPath, Profile profile, CommonParams config,
     Stats stats) {
-    try (Mbtiles output = Mbtiles.newFileDatabase(outputPath)) {
+    try (Mbtiles output = Mbtiles.newWriteToFileDatabase(outputPath)) {
       writeOutput(features, output, () -> FileUtils.fileSize(outputPath), profile, config, stats);
     } catch (IOException e) {
       throw new IllegalStateException("Unable to write to " + outputPath, e);
@@ -100,7 +100,6 @@ public class MbtilesWriter {
 
   private void tileWriter(Supplier<Mbtiles.TileEntry> tiles) throws Exception {
     db.setupSchema();
-    db.tuneForWrites();
     if (!config.deferIndexCreation()) {
       db.addIndex();
     } else {

src/test/java/com/onthegomap/flatmap/write/MbtilesTest.java

@@ -10,10 +10,12 @@ import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.geo.TileCoord;
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -26,9 +28,7 @@ public class MbtilesTest {
   public void testWriteTiles(int howMany, boolean deferIndexCreation, boolean optimize)
     throws IOException, SQLException {
     try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
-      db
-        .setupSchema()
-        .tuneForWrites();
+      db.setupSchema();
       if (!deferIndexCreation) {
         db.addIndex();
       }
@@ -55,6 +55,8 @@ public class MbtilesTest {
       var all = TestUtils.getAllTiles(db);
       assertEquals(howMany, all.size());
       assertEquals(expected, all);
+      assertEquals(expected.stream().map(Mbtiles.TileEntry::tile).collect(Collectors.toSet()),
+        new HashSet<>(db.getAllTileCoords()));
       for (var expectedEntry : expected) {
         var tile = expectedEntry.tile();
         byte[] data = db.getTile(tile.x(), tile.y(), tile.z());
@@ -83,7 +85,7 @@ public class MbtilesTest {
   public void testAddMetadata() throws IOException {
     Map<String, String> expected = new TreeMap<>();
     try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
-      var metadata = db.setupSchema().tuneForWrites().metadata();
+      var metadata = db.setupSchema().metadata();
       metadata.setName("name value");
       expected.put("name", "name value");
 
@@ -120,7 +122,7 @@ public class MbtilesTest {
   public void testAddMetadataWorldBounds() throws IOException {
     Map<String, String> expected = new TreeMap<>();
     try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
-      var metadata = db.setupSchema().tuneForWrites().metadata();
+      var metadata = db.setupSchema().metadata();
       metadata.setBoundsAndCenter(GeoUtils.WORLD_LAT_LON_BOUNDS);
       expected.put("bounds", "-180,-85.05113,180,85.05113");
       expected.put("center", "0,0,0");
@@ -133,7 +135,7 @@ public class MbtilesTest {
   public void testAddMetadataSmallBounds() throws IOException {
     Map<String, String> expected = new TreeMap<>();
     try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
-      var metadata = db.setupSchema().tuneForWrites().metadata();
+      var metadata = db.setupSchema().metadata();
       metadata.setBoundsAndCenter(new Envelope(-73.6632, -69.7598, 41.1274, 43.0185));
       expected.put("bounds", "-73.6632,41.1274,-69.7598,43.0185");
       expected.put("center", "-71.7115,42.07295,7");
@@ -144,7 +146,7 @@ public class MbtilesTest {
 
   private void testMetadataJson(Mbtiles.MetadataJson object, String expected) throws IOException {
     try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
-      var metadata = db.setupSchema().tuneForWrites().metadata();
+      var metadata = db.setupSchema().metadata();
       metadata.setJson(object);
       var actual = metadata.getAll().get("json");
       assertSameJson(expected, actual);