Async support for prepare_jinja2_environment, closes #1809

datasette/app.py

@@ -208,6 +208,7 @@ class Datasette:
         crossdb=False,
         nolock=False,
     ):
+        self._startup_invoked = False
         assert config_dir is None or isinstance(
             config_dir, Path
         ), "config_dir= should be a pathlib.Path"
@@ -344,9 +345,6 @@ class Datasette:
         self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus
         self.jinja_env.filters["escape_sqlite"] = escape_sqlite
         self.jinja_env.filters["to_css_class"] = to_css_class
-        # pylint: disable=no-member
-        pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self)
-
         self._register_renderers()
         self._permission_checks = collections.deque(maxlen=200)
         self._root_token = secrets.token_hex(32)
@@ -389,8 +387,16 @@ class Datasette:
         return Urls(self)
 
     async def invoke_startup(self):
+        # This must be called for Datasette to be in a usable state
+        if self._startup_invoked:
+            return
+        for hook in pm.hook.prepare_jinja2_environment(
+            env=self.jinja_env, datasette=self
+        ):
+            await await_me_maybe(hook)
         for hook in pm.hook.startup(datasette=self):
             await await_me_maybe(hook)
+        self._startup_invoked = True
 
     def sign(self, value, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).dumps(value)
@@ -933,6 +939,8 @@ class Datasette:
     async def render_template(
         self, templates, context=None, request=None, view_name=None
     ):
+        if not self._startup_invoked:
+            raise Exception("render_template() called before await ds.invoke_startup()")
         context = context or {}
         if isinstance(templates, Template):
             template = templates
@@ -1495,34 +1503,42 @@ class DatasetteClient:
         return path
 
     async def get(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.get(self._fix(path), **kwargs)
 
     async def options(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.options(self._fix(path), **kwargs)
 
     async def head(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.head(self._fix(path), **kwargs)
 
     async def post(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.post(self._fix(path), **kwargs)
 
     async def put(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.put(self._fix(path), **kwargs)
 
     async def patch(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.patch(self._fix(path), **kwargs)
 
     async def delete(self, path, **kwargs):
+        await self.ds.invoke_startup()
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.delete(self._fix(path), **kwargs)
 
     async def request(self, method, path, **kwargs):
+        await self.ds.invoke_startup()
         avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None)
         async with httpx.AsyncClient(app=self.app) as client:
             return await client.request(

datasette/utils/testing.py

@@ -147,6 +147,7 @@ class TestClient:
         content_type=None,
         if_none_match=None,
     ):
+        await self.ds.invoke_startup()
         headers = headers or {}
         if content_type:
             headers["content-type"] = content_type

docs/plugin_hooks.rst

@@ -88,6 +88,8 @@ You can now use this filter in your custom templates like so::
 
     Table name: {{ table|uppercase }}
 
+This function can return an awaitable function if it needs to run any async code.
+
 Examples: `datasette-edit-templates <https://datasette.io/plugins/datasette-edit-templates>`_
 
 .. _plugin_hook_extra_template_vars:

docs/testing_plugins.rst

@@ -52,6 +52,36 @@ Then run the tests using pytest like so::
 
     pytest
 
+.. _testing_plugins_datasette_test_instance:
+
+Setting up a Datasette test instance
+------------------------------------
+
+The above example shows the easiest way to start writing tests against a Datasette instance:
+
+.. code-block:: python
+
+    from datasette.app import Datasette
+    import pytest
+
+
+    @pytest.mark.asyncio
+    async def test_plugin_is_installed():
+        datasette = Datasette(memory=True)
+        response = await datasette.client.get("/-/plugins.json")
+        assert response.status_code == 200
+
+Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this:
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.invoke_startup()
+
+This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
+
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you.
+
 .. _testing_plugins_pdb:
 
 Using pdb for errors thrown inside Datasette

tests/fixtures.py

@@ -71,6 +71,7 @@ EXPECTED_PLUGINS = [
             "handle_exception",
             "menu_links",
             "permission_allowed",
+            "prepare_jinja2_environment",
             "register_routes",
             "render_cell",
             "startup",

tests/plugins/my_plugin.py

@@ -143,8 +143,14 @@ def extra_template_vars(
 
 @hookimpl
 def prepare_jinja2_environment(env, datasette):
-    env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}"
-    env.filters["to_hello"] = lambda s: datasette._HELLO
+    async def select_times_three(s):
+        db = datasette.get_database()
+        return (await db.execute("select 3 * ?", [int(s)])).first()[0]
+
+    async def inner():
+        env.filters["select_times_three"] = select_times_three
+
+    return inner
 
 
 @hookimpl

tests/plugins/my_plugin_2.py

@@ -126,6 +126,12 @@ def permission_allowed(datasette, actor, action):
     return inner
 
 
+@hookimpl
+def prepare_jinja2_environment(env, datasette):
+    env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}"
+    env.filters["to_hello"] = lambda s: datasette._HELLO
+
+
 @hookimpl
 def startup(datasette):
     async def inner():

tests/test_internals_datasette_client.py

@@ -1,10 +1,12 @@
 from .fixtures import app_client
 import httpx
 import pytest
+import pytest_asyncio
 
 
-@pytest.fixture
-def datasette(app_client):
+@pytest_asyncio.fixture
+async def datasette(app_client):
+    await app_client.ds.invoke_startup()
     return app_client.ds
 
 

tests/test_plugins.py

@@ -546,11 +546,13 @@ def test_hook_register_output_renderer_can_render(app_client):
 @pytest.mark.asyncio
 async def test_hook_prepare_jinja2_environment(app_client):
     app_client.ds._HELLO = "HI"
+    await app_client.ds.invoke_startup()
     template = app_client.ds.jinja_env.from_string(
-        "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341}
+        "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}",
+        {"a": 3412341, "b": 5},
     )
     rendered = await app_client.ds.render_template(template)
-    assert "Hello there, 3,412,341, HI" == rendered
+    assert "Hello there, 3,412,341, HI, 15" == rendered
 
 
 def test_hook_publish_subcommand():

tests/test_routes.py

@@ -59,6 +59,7 @@ def test_routes(routes, path, expected_class, expected_matches):
 @pytest_asyncio.fixture
 async def ds_with_route():
     ds = Datasette()
+    await ds.invoke_startup()
     ds.remove_database("_memory")
     db = Database(ds, is_memory=True, memory_name="route-name-db")
     ds.add_database(db, name="original-name", route="custom-route-name")