From 5e9895c67f08e9f42acedd3d6d29512ac446e15f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Dec 2020 17:01:18 -0800 Subject: [PATCH] Database(memory_name=) for shared in-memory databases, closes #1151 --- datasette/database.py | 24 +++++++++++++++++++-- docs/internals.rst | 37 +++++++++++++++++++++++++++++--- tests/test_internals_database.py | 30 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 412e0c59..a977b362 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -24,11 +24,18 @@ connections = threading.local() class Database: - def __init__(self, ds, path=None, is_mutable=False, is_memory=False): + def __init__( + self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + ): self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory + self.memory_name = memory_name + if memory_name is not None: + self.path = memory_name + self.is_memory = True + self.is_mutable = True self.hash = None self.cached_size = None self.cached_table_counts = None @@ -46,6 +53,16 @@ class Database: } def connect(self, write=False): + if self.memory_name: + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) + conn = sqlite3.connect( + uri, + uri=True, + check_same_thread=False, + ) + if not write: + conn.execute("PRAGMA query_only=1") + return conn if self.is_memory: return sqlite3.connect(":memory:") # mode=ro or immutable=1? @@ -215,7 +232,10 @@ class Database: @property def name(self): if self.is_memory: - return ":memory:" + if self.memory_name: + return ":memory:{}".format(self.memory_name) + else: + return ":memory:" else: return Path(self.path).stem diff --git a/docs/internals.rst b/docs/internals.rst index ff566f69..b68a1d8a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -270,11 +270,16 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` This will add a mutable database from the provided file path. -The ``Database()`` constructor takes four arguments: the first is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. +To create a shared in-memory database named ``statistics``, use the following: -Use ``is_mutable`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. +.. code-block:: python -Use ``is_memory`` if the connection is to an in-memory SQLite database. + from datasette.database import Database + + datasette.add_database("statistics", Database( + datasette, + memory_name="statistics" + )) .. _datasette_remove_database: @@ -480,6 +485,32 @@ Database class Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas. +.. _database_constructor: + +Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) +---------------------------------------------------------------------------- + +The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. + +The arguments are as follows: + +``ds`` - :ref:`internals_datasette` (required) + The Datasette instance you are attaching this database to. + +``path`` - string + Path to a SQLite database file on disk. + +``is_mutable`` - boolean + Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + +``is_memory`` - boolean + Use this to create non-shared memory connections. + +``memory_name`` - string or ``None`` + Use this to create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads and will persist an changes made to them for the lifetime of the Datasette server process. + +The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. + .. _database_execute: await db.execute(sql, ...) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 49b8a1b3..dc1af48c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -464,3 +464,33 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False + + +@pytest.mark.asyncio +async def test_database_memory_name(app_client): + ds = app_client.ds + foo1 = Database(ds, memory_name="foo") + foo2 = Database(ds, memory_name="foo") + bar1 = Database(ds, memory_name="bar") + bar2 = Database(ds, memory_name="bar") + for db in (foo1, foo2, bar1, bar2): + table_names = await db.table_names() + assert table_names == [] + # Now create a table in foo + await foo1.execute_write("create table foo (t text)", block=True) + assert await foo1.table_names() == ["foo"] + assert await foo2.table_names() == ["foo"] + assert await bar1.table_names() == [] + assert await bar2.table_names() == [] + + +@pytest.mark.asyncio +async def test_in_memory_databases_forbid_writes(app_client): + ds = app_client.ds + db = Database(ds, memory_name="test") + with pytest.raises(sqlite3.OperationalError): + await db.execute("create table foo (t text)") + assert await db.table_names() == [] + # Using db.execute_write() should work: + await db.execute_write("create table foo (t text)", block=True) + assert await db.table_names() == ["foo"]