datasette, not self.ds, in internals documentation

Closes #2319

datasette/app.py

@@ -930,7 +930,7 @@ class Datasette:
             used_default = True
         self._permission_checks.append(
             {
-                "when": datetime.datetime.utcnow().isoformat(),
+                "when": datetime.datetime.now(datetime.timezone.utc).isoformat(),
                 "actor": actor,
                 "action": action,
                 "resource": resource,
@@ -1933,37 +1933,40 @@ class DatasetteClient:
             path = f"http://localhost{path}"
         return path
 
+    async def _request(self, method, path, **kwargs):
+        async with httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=self.app),
+            cookies=kwargs.pop("cookies", None),
+        ) as client:
+            return await getattr(client, method)(self._fix(path), **kwargs)
+
     async def get(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.get(self._fix(path), **kwargs)
+        return await self._request("get", path, **kwargs)
 
     async def options(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.options(self._fix(path), **kwargs)
+        return await self._request("options", path, **kwargs)
 
     async def head(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.head(self._fix(path), **kwargs)
+        return await self._request("head", path, **kwargs)
 
     async def post(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.post(self._fix(path), **kwargs)
+        return await self._request("post", path, **kwargs)
 
     async def put(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.put(self._fix(path), **kwargs)
+        return await self._request("put", path, **kwargs)
 
     async def patch(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.patch(self._fix(path), **kwargs)
+        return await self._request("patch", path, **kwargs)
 
     async def delete(self, path, **kwargs):
-        async with httpx.AsyncClient(app=self.app) as client:
-            return await client.delete(self._fix(path), **kwargs)
+        return await self._request("delete", path, **kwargs)
 
     async def request(self, method, path, **kwargs):
         avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None)
-        async with httpx.AsyncClient(app=self.app) as client:
+        async with httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=self.app),
+            cookies=kwargs.pop("cookies", None),
+        ) as client:
             return await client.request(
                 method, self._fix(path, avoid_path_rewrites), **kwargs
             )

datasette/default_magic_parameters.py

@@ -24,9 +24,12 @@ def now(key, request):
     if key == "epoch":
         return int(time.time())
     elif key == "date_utc":
-        return datetime.datetime.utcnow().date().isoformat()
+        return datetime.datetime.now(datetime.timezone.utc).date().isoformat()
     elif key == "datetime_utc":
-        return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z"
+        return (
+            datetime.datetime.now(datetime.timezone.utc).strftime(r"%Y-%m-%dT%H:%M:%S")
+            + "Z"
+        )
     else:
         raise KeyError
 

datasette/static/app.css

@@ -375,6 +375,7 @@ form.nav-menu-logout {
     position: absolute;
     top: calc(100% + 10px);
     left: 0;
+    z-index: 10000;
 }
 .page-action-menu .icon-text {
     display: inline-flex;

datasette/utils/__init__.py

@@ -1,5 +1,6 @@
 import asyncio
 from contextlib import contextmanager
+import aiofiles
 import click
 from collections import OrderedDict, namedtuple, Counter
 import copy
@@ -1418,3 +1419,24 @@ def md5_not_usedforsecurity(s):
     except TypeError:
         # For Python 3.8 which does not support usedforsecurity=False
         return hashlib.md5(s.encode("utf8")).hexdigest()
+
+
+_etag_cache = {}
+
+
+async def calculate_etag(filepath, chunk_size=4096):
+    if filepath in _etag_cache:
+        return _etag_cache[filepath]
+
+    hasher = hashlib.md5()
+    async with aiofiles.open(filepath, "rb") as f:
+        while True:
+            chunk = await f.read(chunk_size)
+            if not chunk:
+                break
+            hasher.update(chunk)
+
+    etag = f'"{hasher.hexdigest()}"'
+    _etag_cache[filepath] = etag
+
+    return etag

datasette/utils/asgi.py

@@ -1,5 +1,6 @@
+import hashlib
 import json
-from datasette.utils import MultiParams
+from datasette.utils import MultiParams, calculate_etag
 from mimetypes import guess_type
 from urllib.parse import parse_qs, urlunparse, parse_qsl
 from pathlib import Path
@@ -285,6 +286,7 @@ async def asgi_send_file(
     headers = headers or {}
     if filename:
         headers["content-disposition"] = f'attachment; filename="{filename}"'
+
     first = True
     headers["content-length"] = str((await aiofiles.os.stat(str(filepath))).st_size)
     async with aiofiles.open(str(filepath), mode="rb") as fp:
@@ -307,9 +309,14 @@ async def asgi_send_file(
 
 def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
     root_path = Path(root_path)
+    static_headers = {}
+
+    if headers:
+        static_headers = headers.copy()
 
     async def inner_static(request, send):
         path = request.scope["url_route"]["kwargs"]["path"]
+        headers = static_headers.copy()
         try:
             full_path = (root_path / path).resolve().absolute()
         except FileNotFoundError:
@@ -325,7 +332,15 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: Path not inside root path", 404)
             return
         try:
-            await asgi_send_file(send, full_path, chunk_size=chunk_size)
+            # Calculate ETag for filepath
+            etag = await calculate_etag(full_path, chunk_size=chunk_size)
+            headers["ETag"] = etag
+            if_none_match = request.headers.get("if-none-match")
+            if if_none_match and if_none_match == etag:
+                return await asgi_send(send, "", 304)
+            await asgi_send_file(
+                send, full_path, chunk_size=chunk_size, headers=headers
+            )
         except FileNotFoundError:
             await asgi_send_html(send, "404: File not found", 404)
             return

docs/authentication.rst

@@ -312,7 +312,7 @@ If you want to provide access to any actor with a value for a specific key, use
         }
 .. [[[end]]]
 
-You can specify that only unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo <https://latest.datasette.io/-/allow-debug?actor=null&allow=%7B%0D%0A++++%22unauthenticated%22%3A+true%0D%0A%7D>`__, `deny demo <https://latest.datasette.io/-/allow-debug?actor=%7B%0D%0A++++%22id%22%3A+%22hello%22%0D%0A%7D&allow=%7B%0D%0A++++%22unauthenticated%22%3A+true%0D%0A%7D>`__):
+You can specify that only unauthenticated actors (from anonymous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo <https://latest.datasette.io/-/allow-debug?actor=null&allow=%7B%0D%0A++++%22unauthenticated%22%3A+true%0D%0A%7D>`__, `deny demo <https://latest.datasette.io/-/allow-debug?actor=%7B%0D%0A++++%22id%22%3A+%22hello%22%0D%0A%7D&allow=%7B%0D%0A++++%22unauthenticated%22%3A+true%0D%0A%7D>`__):
 
 .. [[[cog
     from metadata_doc import config_example

docs/changelog.rst

@@ -14,10 +14,12 @@ Each of the key concepts in Datasette now has an :ref:`actions menu <plugin_acti
 - Plugin hook: :ref:`view_actions() <plugin_hook_view_actions>` for actions that can be applied to a SQL view. (:issue:`2297`)
 - Plugin hook: :ref:`homepage_actions() <plugin_hook_homepage_actions>` for actions that apply to the instance homepage. (:issue:`2298`)
 - Plugin hook: :ref:`row_actions() <plugin_hook_row_actions>` for actions that apply to the row page. (:issue:`2299`)
+- Action menu items for all of the ``*_actions()`` plugin hooks can now return an optional ``"description"`` key, which will be displayed in the menu below the action label. (:issue:`2294`)
 - :ref:`Plugin hooks <plugin_hooks>` documentation page is now organized with additional headings. (:issue:`2300`)
 - Improved the display of action buttons on pages that also display metadata. (:issue:`2286`)
 - The header and footer of the page now uses a subtle gradient effect, and options in the navigation menu are better visually defined. (:issue:`2302`)
 - Table names that start with an underscore now default to hidden. (:issue:`2104`)
+- ``pragma_table_list`` has been added to the allow-list of SQLite pragma functions supported by Datasette. ``select * from pragma_table_list()`` is no longer blocked. (`#2104 <https://github.com/simonw/datasette/issues/2104#issuecomment-1982352475>`__)
 
 .. _v1_0_a12:
 

docs/internals.rst

@@ -386,7 +386,7 @@ This is useful when you need to check multiple permissions at once. For example,
 
 .. code-block:: python
 
-    await self.ds.ensure_permissions(
+    await datasette.ensure_permissions(
         request.actor,
         [
             ("view-table", (database, table)),
@@ -420,7 +420,7 @@ This example checks if the user can access a specific table, and sets ``private`
 
 .. code-block:: python
 
-    visible, private = await self.ds.check_visibility(
+    visible, private = await datasette.check_visibility(
         request.actor,
         action="view-table",
         resource=(database, table),
@@ -430,7 +430,7 @@ The following example runs three checks in a row, similar to :ref:`datasette_ens
 
 .. code-block:: python
 
-    visible, private = await self.ds.check_visibility(
+    visible, private = await datasette.check_visibility(
         request.actor,
         permissions=[
             ("view-table", (database, table)),
@@ -1222,7 +1222,7 @@ Plugins can access this database by calling ``internal_db = datasette.get_intern
 
 Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example:
 
-1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``.
+1. Use a unique prefix when creating tables, indices, and triggers in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``.
 2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time.
 3. Use temporary tables or shared in-memory attached databases when possible.
 4. Avoid implementing features that could expose private data stored in the internal database by other plugins.
@@ -1234,7 +1234,7 @@ The datasette.utils module
 
 The ``datasette.utils`` module contains various utility functions used by Datasette. As a general rule you should consider anything in this module to be unstable - functions and classes here could change without warning or be removed entirely between Datasette releases, without being mentioned in the release notes.
 
-The exception to this rule is anythang that is documented here. If you find a need for an undocumented utility function in your own work, consider `opening an issue <https://github.com/simonw/datasette/issues/new>`__ requesting that the function you are using be upgraded to documented and supported status.
+The exception to this rule is anything that is documented here. If you find a need for an undocumented utility function in your own work, consider `opening an issue <https://github.com/simonw/datasette/issues/new>`__ requesting that the function you are using be upgraded to documented and supported status.
 
 .. _internals_utils_parse_metadata:
 

docs/json_api.rst

@@ -568,6 +568,8 @@ To insert multiple rows at a time, use the same API method but send a list of di
 
 If successful, this will return a ``201`` status code and a ``{"ok": true}`` response body.
 
+The maximum number rows that can be submitted at once defaults to 100, but this can be changed using the :ref:`setting_max_insert_rows` setting.
+
 To return the newly inserted rows, add the ``"return": true`` key to the request body:
 
 .. code-block:: json

docs/plugin_hooks.rst

@@ -494,7 +494,7 @@ This will register ``render_demo`` to be called when paths with the extension ``
 
 ``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls.
 
-``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin.
+``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influence if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin.
 
 When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature.
 
@@ -1729,6 +1729,8 @@ This example displays the row in JSON plus some additional debug information if
                 },
             ]
 
+Example: `datasette-enrichments <https://datasette.io/plugins/datasette-enrichments>`_
+
 .. _plugin_hook_database_actions:
 
 database_actions(datasette, actor, database, request)
@@ -1991,6 +1993,49 @@ This example plugin logs details of all events to standard error:
         )
         print(msg, file=sys.stderr, flush=True)
 
+The function can also return an async function which will be awaited. This is useful for writing to a database.
+
+This example logs events to a `datasette_events` table in a database called `events`. It uses the `startup()` hook to create that table if it does not exist.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import json
+
+    @hookimpl
+    def startup(datasette):
+        async def inner():
+            db = datasette.get_database("events")
+            await db.execute_write(
+                """
+                create table if not exists datasette_events (
+                    id integer primary key,
+                    event_type text,
+                    created text,
+                    actor text,
+                    properties text
+                )
+            """
+            )
+
+        return inner
+
+
+    @hookimpl
+    def track_event(datasette, event):
+        async def inner():
+            db = datasette.get_database("events")
+            properties = event.properties()
+            await db.execute_write(
+                """
+                insert into datasette_events (event_type, created, actor, properties)
+                values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'), ?, ?)
+            """,
+                (event.name, json.dumps(event.actor), json.dumps(properties)),
+            )
+
+        return inner
+
 Example: `datasette-events-db <https://datasette.io/plugins/datasette-events-db>`_
 
 .. _plugin_hook_register_events:

tests/test_html.py

@@ -78,6 +78,10 @@ async def test_static(ds_client):
     response = await ds_client.get("/-/static/app.css")
     assert response.status_code == 200
     assert "text/css" == response.headers["content-type"]
+    assert "etag" in response.headers
+    etag = response.headers.get("etag")
+    response = await ds_client.get("/-/static/app.css", headers={"if-none-match": etag})
+    assert response.status_code == 304
 
 
 def test_static_mounts():

tests/test_utils.py

@@ -706,3 +706,15 @@ def test_truncate_url(url, length, expected):
 def test_pairs_to_nested_config(pairs, expected):
     actual = utils.pairs_to_nested_config(pairs)
     assert actual == expected
+
+
+@pytest.mark.asyncio
+async def test_calculate_etag(tmp_path):
+    path = tmp_path / "test.txt"
+    path.write_text("hello")
+    etag = '"5d41402abc4b2a76b9719d911017c592"'
+    assert etag == await utils.calculate_etag(path)
+    assert utils._etag_cache[path] == etag
+    utils._etag_cache[path] = "hash"
+    assert "hash" == await utils.calculate_etag(path)
+    utils._etag_cache.clear()