From 5fac26aa221a111d7633f2dd92014641f7c0ade9 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 19 Dec 2021 12:54:12 -0800
Subject: [PATCH] Another populate_schema_tables optimization, refs #1555

---
 datasette/utils/internal_db.py | 41 +++++++++-------------------------
 tests/test_api.py              |  1 -
 2 files changed, 11 insertions(+), 31 deletions(-)

diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py
index 58f99825..ed589a7a 100644
--- a/datasette/utils/internal_db.py
+++ b/datasette/utils/internal_db.py
@@ -67,18 +67,23 @@ async def init_internal_db(db):
 
 async def populate_schema_tables(internal_db, db):
     database_name = db.name
-    await internal_db.execute_write(
-        "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
-    )
+
+    def delete_everything(conn):
+        conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name])
+        conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name])
+        conn.execute(
+            "DELETE FROM foreign_keys WHERE database_name = ?", [database_name]
+        )
+        conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name])
+
+    await internal_db.execute_write_fn(delete_everything, block=True)
+
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
 
     def collect_info(conn):
         tables_to_insert = []
-        columns_to_delete = []
         columns_to_insert = []
-        foreign_keys_to_delete = []
         foreign_keys_to_insert = []
-        indexes_to_delete = []
         indexes_to_insert = []
 
         for table in tables:
@@ -86,7 +91,6 @@ async def populate_schema_tables(internal_db, db):
             tables_to_insert.append(
                 (database_name, table_name, table["rootpage"], table["sql"])
             )
-            columns_to_delete.append((database_name, table_name))
             columns = table_column_details(conn, table_name)
             columns_to_insert.extend(
                 {
@@ -95,7 +99,6 @@ async def populate_schema_tables(internal_db, db):
                 }
                 for column in columns
             )
-            foreign_keys_to_delete.append((database_name, table_name))
             foreign_keys = conn.execute(
                 f"PRAGMA foreign_key_list([{table_name}])"
             ).fetchall()
@@ -106,7 +109,6 @@ async def populate_schema_tables(internal_db, db):
                 }
                 for foreign_key in foreign_keys
             )
-            indexes_to_delete.append((database_name, table_name))
             indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall()
             indexes_to_insert.extend(
                 {
@@ -117,21 +119,15 @@ async def populate_schema_tables(internal_db, db):
             )
         return (
             tables_to_insert,
-            columns_to_delete,
             columns_to_insert,
-            foreign_keys_to_delete,
             foreign_keys_to_insert,
-            indexes_to_delete,
             indexes_to_insert,
         )
 
     (
         tables_to_insert,
-        columns_to_delete,
         columns_to_insert,
-        foreign_keys_to_delete,
         foreign_keys_to_insert,
-        indexes_to_delete,
         indexes_to_insert,
     ) = await db.execute_fn(collect_info)
 
@@ -143,11 +139,6 @@ async def populate_schema_tables(internal_db, db):
         tables_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM columns WHERE database_name = ? and table_name = ?",
-        columns_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO columns (
@@ -159,11 +150,6 @@ async def populate_schema_tables(internal_db, db):
         columns_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
-        foreign_keys_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO foreign_keys (
@@ -175,11 +161,6 @@ async def populate_schema_tables(internal_db, db):
         foreign_keys_to_insert,
         block=True,
     )
-    await internal_db.execute_write_many(
-        "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
-        indexes_to_delete,
-        block=True,
-    )
     await internal_db.execute_write_many(
         """
         INSERT INTO indexes (
diff --git a/tests/test_api.py b/tests/test_api.py
index 8ecaef43..574ebb41 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -946,7 +946,6 @@ def test_trace(trace_debug):
         "CREATE TABLE ",
         "PRAGMA ",
         "INSERT OR REPLACE INTO ",
-        "DELETE FROM ",
         "INSERT INTO",
         "select ",
     )