diff --git a/datasette/database.py b/datasette/database.py index ab3c82c9..ffa7a794 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -132,7 +132,7 @@ class Database: with sqlite_timelimit(conn, time_limit_ms): try: cursor = conn.cursor() - cursor.execute(sql, params or {}) + cursor.execute(sql, params if params is not None else {}) max_returned_rows = self.ds.max_returned_rows if max_returned_rows == page_size: max_returned_rows += 1 diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py new file mode 100644 index 00000000..ac7c5eac --- /dev/null +++ b/datasette/default_magic_parameters.py @@ -0,0 +1,55 @@ +from datasette import hookimpl +from datasette.utils import escape_fts +import datetime +import os +import time + + +def header(key, request): + key = key.replace("_", "-").encode("utf-8") + headers_dict = dict(request.scope["headers"]) + return headers_dict[key].decode("utf-8") + + +def actor(key, request): + if request.actor is None: + raise KeyError + return request.actor[key] + + +def cookie(key, request): + return request.cookies[key] + + +def timestamp(key, request): + if key == "epoch": + return int(time.time()) + elif key == "date_utc": + return datetime.datetime.utcnow().date().isoformat() + elif key == "datetime_utc": + return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z" + else: + raise KeyError + + +def random(key, request): + if key.startswith("chars_") and key.split("chars_")[-1].isdigit(): + num_chars = int(key.split("chars_")[-1]) + if num_chars % 2 == 1: + urandom_len = (num_chars + 1) / 2 + else: + urandom_len = num_chars / 2 + return os.urandom(int(urandom_len)).hex()[:num_chars] + else: + raise KeyError + + +@hookimpl +def register_magic_parameters(): + return [ + ("header", header), + ("actor", actor), + ("cookie", cookie), + ("timestamp", timestamp), + ("random", random), + ] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 91feb49b..020e84b9 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -83,3 +83,8 @@ def permission_allowed(datasette, actor, action, resource): @hookspec def canned_queries(datasette, database, actor): "Return a dictonary of canned query definitions or an awaitable function that returns them" + + +@hookspec +def register_magic_parameters(datasette): + "Return a list of (name, function) magic parameter functions" diff --git a/datasette/plugins.py b/datasette/plugins.py index b35b750f..cb3d2c34 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -11,6 +11,7 @@ DEFAULT_PLUGINS = ( "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", + "datasette.default_magic_parameters", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 14060669..30121cf2 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -4,6 +4,7 @@ import base64 import click import hashlib import inspect +import itertools import json import mergedeep import os @@ -17,6 +18,7 @@ import urllib import numbers import yaml from .shutil_backport import copytree +from ..plugins import pm try: import pysqlite3 as sqlite3 diff --git a/datasette/views/database.py b/datasette/views/database.py index ad28fb63..44750f5b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,5 @@ import os +import itertools import jinja2 from datasette.utils import ( @@ -165,11 +166,12 @@ class QueryView(DataView): named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters + if not named_parameter.startswith("_") } # Set to blank string if missing from params for named_parameter in named_parameters: - if named_parameter not in params: + if named_parameter not in params and not named_parameter.startswith("_"): params[named_parameter] = "" extra_args = {} @@ -184,9 +186,13 @@ class QueryView(DataView): if write: if request.method == "POST": params = await request.post_vars() + if canned_query: + params_for_query = MagicParameters(params, request, self.ds) + else: + params_for_query = params try: cursor = await self.ds.databases[database].execute_write( - sql, params, block=True + sql, params_for_query, block=True ) message = metadata.get( "on_success_message" @@ -227,8 +233,12 @@ class QueryView(DataView): templates, ) else: # Not a write + if canned_query: + params_for_query = MagicParameters(params, request, self.ds) + else: + params_for_query = params results = await self.ds.execute( - database, sql, params, truncate=True, **extra_args + database, sql, params_for_query, truncate=True, **extra_args ) columns = [r[0] for r in results.description] @@ -298,3 +308,25 @@ class QueryView(DataView): extra_template, templates, ) + + +class MagicParameters(dict): + def __init__(self, data, request, datasette): + super().__init__(data) + self._request = request + self._magics = dict( + itertools.chain.from_iterable( + pm.hook.register_magic_parameters(datasette=datasette) + ) + ) + + def __getitem__(self, key): + if key.startswith("_") and key.count("_") >= 2: + prefix, suffix = key[1:].split("_", 1) + if prefix in self._magics: + try: + return self._magics[prefix](suffix, self._request) + except KeyError: + return super().__getitem__(key) + else: + return super().__getitem__(key) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4e118bdf..8683bee8 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -895,3 +895,42 @@ Here's an example that allows users to view the ``admin_log`` table only if thei See :ref:`built-in permissions ` for a full list of permissions that are included in Datasette core. Example: `datasette-permissions-sql `_ + +.. _plugin_hook_register_magic_parameters: + +register_magic_parameters(datasette) +------------------------------------ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries `. This plugin hook allows additional magic parameters to be defined by plugins. + +Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. + +To register a new function, return it as a tuple of ``(string prefix, function)`` from this hook. The function you register should take two arguments: ``key`` and ``request``, where ``key`` is the ``rest_of_parameter`` portion of the parameter and ``request`` is the current :ref:`internals_request`. + +This example registers two new magic parameters: ``:_request_http_version`` returning the HTTP version of the current request, and ``:_uuid_new`` which returns a new UUID: + +.. code-block:: python + + from uuid import uuid4 + + def uuid(key, request): + if key == "new": + return str(uuid4()) + else: + raise KeyError + + def request(key, request): + if key == "http_version": + return request.scope["http_version"] + else: + raise KeyError + + @hookimpl + def register_magic_parameters(datasette): + return [ + ("request", request), + ("uuid", uuid), + ] diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 6cc32da1..aff16c1a 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -114,8 +114,8 @@ rendered as HTML (rather than having HTML special characters escaped). .. _canned_queries_named_parameters: -Named parameters -~~~~~~~~~~~~~~~~ +Canned query parameters +~~~~~~~~~~~~~~~~~~~~~~~ Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page @@ -274,6 +274,58 @@ You can use ``"params"`` to explicitly list the named parameters that should be You can pre-populate form fields when the page first loads using a querystring, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query. +.. _canned_queries_magic_parameters: + +Magic parameters +~~~~~~~~~~~~~~~~ + +Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or querystring. + +Available magic parameters are: + +``_actor_*`` - e.g. ``_actor_id``, ``_actor_name`` + Fields from the currently authenticated :ref:`authentication_actor`. + +``_header_*`` - e.g. ``_header_user_agent`` + Header from the incoming HTTP request. The key should be in lower case and with hyphens converted to underscores e.g. ``_header_user_agent`` or ``_header_accept_language``. + +``_cookie_*`` - e.g. ``_cookie_lang`` + The value of the incoming cookie of that name. + +``_timestamp_epoch`` + The number of seconds since the Unix epoch. + +``_timestamp_date_utc`` + The date in UTC, e.g. ``2020-06-01`` + +``_timestamp_datetime_utc`` + The ISO 8601 datetime in UTC, e.g. ``2020-06-24T18:01:07Z`` + +``_random_chars_*`` - e.g. ``_random_chars_128`` + A random string of characters of the specified length. + +Here's an example configuration (this time using ``metadata.yaml`` since that provides better support for multi-line SQL queries) that adds a message from the authenticated user, storing various pieces of additional metadata using magic parameters: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + add_message: + allow: + id: "*" + sql: |- + INSERT INTO messages ( + user_id, ip, message, datetime + ) VALUES ( + :_actor_id, :_request_ip, :message, :_timestamp_datetime_utc + ) + write: true + +The form presented at ``/mydatabase/add_message`` will have just a field for ``message`` - the other parameters will be populated by the magic parameter mechanism. + +Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. + .. _pagination: Pagination diff --git a/tests/fixtures.py b/tests/fixtures.py index 9b28c283..46cf2ef0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -49,6 +49,7 @@ EXPECTED_PLUGINS = [ "prepare_connection", "prepare_jinja2_environment", "register_facet_classes", + "register_magic_parameters", "register_routes", "render_cell", "startup", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 45eae412..e4e4153c 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -212,3 +212,25 @@ def canned_queries(datasette, database, actor): actor["id"] if actor else "null" ) } + + +@hookimpl +def register_magic_parameters(): + from uuid import uuid4 + + def uuid(key, request): + if key == "new": + return str(uuid4()) + else: + raise KeyError + + def request(key, request): + if key == "http_version": + return request.scope["http_version"] + else: + raise KeyError + + return [ + ("request", request), + ("uuid", uuid), + ] diff --git a/tests/test_api.py b/tests/test_api.py index 322a0001..1f93c1a7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -601,18 +601,6 @@ def test_custom_sql(app_client): assert not data["truncated"] -def test_canned_query_with_named_parameter(app_client): - response = app_client.get("/fixtures/neighborhood_search.json?text=town") - assert [ - ["Corktown", "Detroit", "MI"], - ["Downtown", "Los Angeles", "CA"], - ["Downtown", "Detroit", "MI"], - ["Greektown", "Detroit", "MI"], - ["Koreatown", "Los Angeles", "CA"], - ["Mexicantown", "Detroit", "MI"], - ] == response.json["rows"] - - def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") assert 400 == response.status diff --git a/tests/test_canned_write.py b/tests/test_canned_queries.py similarity index 63% rename from tests/test_canned_write.py rename to tests/test_canned_queries.py index e33eed69..3942dc98 100644 --- a/tests/test_canned_write.py +++ b/tests/test_canned_queries.py @@ -1,5 +1,7 @@ +from bs4 import BeautifulSoup as Soup import pytest -from .fixtures import make_app_client +import re +from .fixtures import make_app_client, app_client @pytest.fixture @@ -39,6 +41,18 @@ def canned_write_client(): yield client +def test_canned_query_with_named_parameter(app_client): + response = app_client.get("/fixtures/neighborhood_search.json?text=town") + assert [ + ["Corktown", "Detroit", "MI"], + ["Downtown", "Los Angeles", "CA"], + ["Downtown", "Detroit", "MI"], + ["Greektown", "Detroit", "MI"], + ["Koreatown", "Los Angeles", "CA"], + ["Mexicantown", "Detroit", "MI"], + ] == response.json["rows"] + + def test_insert(canned_write_client): response = canned_write_client.post( "/data/add_name", {"name": "Hello"}, allow_redirects=False, csrftoken_from=True, @@ -147,3 +161,74 @@ def test_canned_query_permissions(canned_write_client): cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})} assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status + + +@pytest.fixture(scope="session") +def magic_parameters_client(): + with make_app_client( + extra_databases={"data.db": "create table logs (line text)"}, + metadata={ + "databases": { + "data": { + "queries": { + "runme_post": {"sql": "", "write": True}, + "runme_get": {"sql": ""}, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.parametrize( + "magic_parameter,expected_re", + [ + ("_actor_id", "root"), + ("_header_host", "localhost"), + ("_cookie_foo", "bar"), + ("_timestamp_epoch", r"^\d+$"), + ("_timestamp_date_utc", r"^\d{4}-\d{2}-\d{2}$"), + ("_timestamp_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), + ("_random_chars_1", r"^\w$"), + ("_random_chars_10", r"^\w{10}$"), + ], +) +def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): + magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_post"][ + "sql" + ] = "insert into logs (line) values (:{})".format(magic_parameter) + magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_get"][ + "sql" + ] = "select :{} as result".format(magic_parameter) + cookies = { + "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), + "foo": "bar", + } + # Test the GET version + get_response = magic_parameters_client.get( + "/data/runme_get.json?_shape=array", cookies=cookies + ) + get_actual = get_response.json[0]["result"] + assert re.match(expected_re, str(get_actual)) + # Test the form + form_response = magic_parameters_client.get("/data/runme_post") + soup = Soup(form_response.body, "html.parser") + # The magic parameter should not be represented as a form field + assert None is soup.find("input", {"name": magic_parameter}) + # Submit the form to create a log line + response = magic_parameters_client.post( + "/data/runme_post", {}, csrftoken_from=True, cookies=cookies + ) + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert re.match(expected_re, post_actual) + + +def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): + response = magic_parameters_client.get( + "/data.json?sql=select+:_header_host&_shape=array" + ) + assert 500 == response.status + assert "You did not supply a value for binding 1." == response.json["error"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a7736756..b798e52d 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -638,3 +638,31 @@ def test_canned_queries_actor(app_client): assert [{"1": 1, "actor_id": "bot"}] == app_client.get( "/fixtures/from_hook.json?_bot=1&_shape=array" ).json + + +def test_register_magic_parameters(restore_working_directory): + with make_app_client( + extra_databases={"data.db": "create table logs (line text)"}, + metadata={ + "databases": { + "data": { + "queries": { + "runme": { + "sql": "insert into logs (line) values (:_request_http_version)", + "write": True, + }, + "get_uuid": {"sql": "select :_uuid_new",}, + } + } + } + }, + ) as client: + response = client.post("/data/runme", {}, csrftoken_from=True) + assert 200 == response.status + actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json + assert [{"rowid": 1, "line": "1.0"}] == actual + # Now try the GET request against get_uuid + response_get = client.get("/data/get_uuid.json?_shape=array") + assert 200 == response_get.status + new_uuid = response_get.json[0][":_uuid_new"] + assert 4 == new_uuid.count("-")