kopia lustrzana https://github.com/simonw/datasette
Magic parameters for canned queries
Closes #842 Includes a new plugin hook, register_magic_parameters()pull/868/head
Simon Willison
GitHub
rodzic
4b142862f2
commit
563f5a2d3a
|
|
@ -132,7 +132,7 @@ class Database:
|
|||
with sqlite_timelimit(conn, time_limit_ms):
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(sql, params or {})
|
||||
cursor.execute(sql, params if params is not None else {})
|
||||
max_returned_rows = self.ds.max_returned_rows
|
||||
if max_returned_rows == page_size:
|
||||
max_returned_rows += 1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
from datasette import hookimpl
|
||||
from datasette.utils import escape_fts
|
||||
import datetime
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
def header(key, request):
|
||||
key = key.replace("_", "-").encode("utf-8")
|
||||
headers_dict = dict(request.scope["headers"])
|
||||
return headers_dict[key].decode("utf-8")
|
||||
|
||||
|
||||
def actor(key, request):
|
||||
if request.actor is None:
|
||||
raise KeyError
|
||||
return request.actor[key]
|
||||
|
||||
|
||||
def cookie(key, request):
|
||||
return request.cookies[key]
|
||||
|
||||
|
||||
def timestamp(key, request):
|
||||
if key == "epoch":
|
||||
return int(time.time())
|
||||
elif key == "date_utc":
|
||||
return datetime.datetime.utcnow().date().isoformat()
|
||||
elif key == "datetime_utc":
|
||||
return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z"
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
|
||||
def random(key, request):
|
||||
if key.startswith("chars_") and key.split("chars_")[-1].isdigit():
|
||||
num_chars = int(key.split("chars_")[-1])
|
||||
if num_chars % 2 == 1:
|
||||
urandom_len = (num_chars + 1) / 2
|
||||
else:
|
||||
urandom_len = num_chars / 2
|
||||
return os.urandom(int(urandom_len)).hex()[:num_chars]
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
|
||||
@hookimpl
|
||||
def register_magic_parameters():
|
||||
return [
|
||||
("header", header),
|
||||
("actor", actor),
|
||||
("cookie", cookie),
|
||||
("timestamp", timestamp),
|
||||
("random", random),
|
||||
]
|
||||
|
|
@ -83,3 +83,8 @@ def permission_allowed(datasette, actor, action, resource):
|
|||
@hookspec
|
||||
def canned_queries(datasette, database, actor):
|
||||
"Return a dictonary of canned query definitions or an awaitable function that returns them"
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_magic_parameters(datasette):
|
||||
"Return a list of (name, function) magic parameter functions"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ DEFAULT_PLUGINS = (
|
|||
"datasette.sql_functions",
|
||||
"datasette.actor_auth_cookie",
|
||||
"datasette.default_permissions",
|
||||
"datasette.default_magic_parameters",
|
||||
)
|
||||
|
||||
pm = pluggy.PluginManager("datasette")
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import base64
|
|||
import click
|
||||
import hashlib
|
||||
import inspect
|
||||
import itertools
|
||||
import json
|
||||
import mergedeep
|
||||
import os
|
||||
|
|
@ -17,6 +18,7 @@ import urllib
|
|||
import numbers
|
||||
import yaml
|
||||
from .shutil_backport import copytree
|
||||
from ..plugins import pm
|
||||
|
||||
try:
|
||||
import pysqlite3 as sqlite3
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import os
|
||||
import itertools
|
||||
import jinja2
|
||||
|
||||
from datasette.utils import (
|
||||
|
|
@ -165,11 +166,12 @@ class QueryView(DataView):
|
|||
named_parameter_values = {
|
||||
named_parameter: params.get(named_parameter) or ""
|
||||
for named_parameter in named_parameters
|
||||
if not named_parameter.startswith("_")
|
||||
}
|
||||
|
||||
# Set to blank string if missing from params
|
||||
for named_parameter in named_parameters:
|
||||
if named_parameter not in params:
|
||||
if named_parameter not in params and not named_parameter.startswith("_"):
|
||||
params[named_parameter] = ""
|
||||
|
||||
extra_args = {}
|
||||
|
|
@ -184,9 +186,13 @@ class QueryView(DataView):
|
|||
if write:
|
||||
if request.method == "POST":
|
||||
params = await request.post_vars()
|
||||
if canned_query:
|
||||
params_for_query = MagicParameters(params, request, self.ds)
|
||||
else:
|
||||
params_for_query = params
|
||||
try:
|
||||
cursor = await self.ds.databases[database].execute_write(
|
||||
sql, params, block=True
|
||||
sql, params_for_query, block=True
|
||||
)
|
||||
message = metadata.get(
|
||||
"on_success_message"
|
||||
|
|
@ -227,8 +233,12 @@ class QueryView(DataView):
|
|||
templates,
|
||||
)
|
||||
else: # Not a write
|
||||
if canned_query:
|
||||
params_for_query = MagicParameters(params, request, self.ds)
|
||||
else:
|
||||
params_for_query = params
|
||||
results = await self.ds.execute(
|
||||
database, sql, params, truncate=True, **extra_args
|
||||
database, sql, params_for_query, truncate=True, **extra_args
|
||||
)
|
||||
columns = [r[0] for r in results.description]
|
||||
|
||||
|
|
@ -298,3 +308,25 @@ class QueryView(DataView):
|
|||
extra_template,
|
||||
templates,
|
||||
)
|
||||
|
||||
|
||||
class MagicParameters(dict):
|
||||
def __init__(self, data, request, datasette):
|
||||
super().__init__(data)
|
||||
self._request = request
|
||||
self._magics = dict(
|
||||
itertools.chain.from_iterable(
|
||||
pm.hook.register_magic_parameters(datasette=datasette)
|
||||
)
|
||||
)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key.startswith("_") and key.count("_") >= 2:
|
||||
prefix, suffix = key[1:].split("_", 1)
|
||||
if prefix in self._magics:
|
||||
try:
|
||||
return self._magics[prefix](suffix, self._request)
|
||||
except KeyError:
|
||||
return super().__getitem__(key)
|
||||
else:
|
||||
return super().__getitem__(key)
|
||||
|
|
|
|||
|
|
@ -895,3 +895,42 @@ Here's an example that allows users to view the ``admin_log`` table only if thei
|
|||
See :ref:`built-in permissions <permissions>` for a full list of permissions that are included in Datasette core.
|
||||
|
||||
Example: `datasette-permissions-sql <https://github.com/simonw/datasette-permissions-sql>`_
|
||||
|
||||
.. _plugin_hook_register_magic_parameters:
|
||||
|
||||
register_magic_parameters(datasette)
|
||||
------------------------------------
|
||||
|
||||
``datasette`` - :ref:`internals_datasette`
|
||||
You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``.
|
||||
|
||||
:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries <canned_queries>`. This plugin hook allows additional magic parameters to be defined by plugins.
|
||||
|
||||
Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function.
|
||||
|
||||
To register a new function, return it as a tuple of ``(string prefix, function)`` from this hook. The function you register should take two arguments: ``key`` and ``request``, where ``key`` is the ``rest_of_parameter`` portion of the parameter and ``request`` is the current :ref:`internals_request`.
|
||||
|
||||
This example registers two new magic parameters: ``:_request_http_version`` returning the HTTP version of the current request, and ``:_uuid_new`` which returns a new UUID:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
def uuid(key, request):
|
||||
if key == "new":
|
||||
return str(uuid4())
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
def request(key, request):
|
||||
if key == "http_version":
|
||||
return request.scope["http_version"]
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
@hookimpl
|
||||
def register_magic_parameters(datasette):
|
||||
return [
|
||||
("request", request),
|
||||
("uuid", uuid),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -114,8 +114,8 @@ rendered as HTML (rather than having HTML special characters escaped).
|
|||
|
||||
.. _canned_queries_named_parameters:
|
||||
|
||||
Named parameters
|
||||
~~~~~~~~~~~~~~~~
|
||||
Canned query parameters
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Canned queries support named parameters, so if you include those in the SQL you
|
||||
will then be able to enter them using the form fields on the canned query page
|
||||
|
|
@ -274,6 +274,58 @@ You can use ``"params"`` to explicitly list the named parameters that should be
|
|||
|
||||
You can pre-populate form fields when the page first loads using a querystring, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query.
|
||||
|
||||
.. _canned_queries_magic_parameters:
|
||||
|
||||
Magic parameters
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or querystring.
|
||||
|
||||
Available magic parameters are:
|
||||
|
||||
``_actor_*`` - e.g. ``_actor_id``, ``_actor_name``
|
||||
Fields from the currently authenticated :ref:`authentication_actor`.
|
||||
|
||||
``_header_*`` - e.g. ``_header_user_agent``
|
||||
Header from the incoming HTTP request. The key should be in lower case and with hyphens converted to underscores e.g. ``_header_user_agent`` or ``_header_accept_language``.
|
||||
|
||||
``_cookie_*`` - e.g. ``_cookie_lang``
|
||||
The value of the incoming cookie of that name.
|
||||
|
||||
``_timestamp_epoch``
|
||||
The number of seconds since the Unix epoch.
|
||||
|
||||
``_timestamp_date_utc``
|
||||
The date in UTC, e.g. ``2020-06-01``
|
||||
|
||||
``_timestamp_datetime_utc``
|
||||
The ISO 8601 datetime in UTC, e.g. ``2020-06-24T18:01:07Z``
|
||||
|
||||
``_random_chars_*`` - e.g. ``_random_chars_128``
|
||||
A random string of characters of the specified length.
|
||||
|
||||
Here's an example configuration (this time using ``metadata.yaml`` since that provides better support for multi-line SQL queries) that adds a message from the authenticated user, storing various pieces of additional metadata using magic parameters:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
databases:
|
||||
mydatabase:
|
||||
queries:
|
||||
add_message:
|
||||
allow:
|
||||
id: "*"
|
||||
sql: |-
|
||||
INSERT INTO messages (
|
||||
user_id, ip, message, datetime
|
||||
) VALUES (
|
||||
:_actor_id, :_request_ip, :message, :_timestamp_datetime_utc
|
||||
)
|
||||
write: true
|
||||
|
||||
The form presented at ``/mydatabase/add_message`` will have just a field for ``message`` - the other parameters will be populated by the magic parameter mechanism.
|
||||
|
||||
Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook.
|
||||
|
||||
.. _pagination:
|
||||
|
||||
Pagination
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ EXPECTED_PLUGINS = [
|
|||
"prepare_connection",
|
||||
"prepare_jinja2_environment",
|
||||
"register_facet_classes",
|
||||
"register_magic_parameters",
|
||||
"register_routes",
|
||||
"render_cell",
|
||||
"startup",
|
||||
|
|
|
|||
|
|
@ -212,3 +212,25 @@ def canned_queries(datasette, database, actor):
|
|||
actor["id"] if actor else "null"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@hookimpl
|
||||
def register_magic_parameters():
|
||||
from uuid import uuid4
|
||||
|
||||
def uuid(key, request):
|
||||
if key == "new":
|
||||
return str(uuid4())
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
def request(key, request):
|
||||
if key == "http_version":
|
||||
return request.scope["http_version"]
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
return [
|
||||
("request", request),
|
||||
("uuid", uuid),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -601,18 +601,6 @@ def test_custom_sql(app_client):
|
|||
assert not data["truncated"]
|
||||
|
||||
|
||||
def test_canned_query_with_named_parameter(app_client):
|
||||
response = app_client.get("/fixtures/neighborhood_search.json?text=town")
|
||||
assert [
|
||||
["Corktown", "Detroit", "MI"],
|
||||
["Downtown", "Los Angeles", "CA"],
|
||||
["Downtown", "Detroit", "MI"],
|
||||
["Greektown", "Detroit", "MI"],
|
||||
["Koreatown", "Los Angeles", "CA"],
|
||||
["Mexicantown", "Detroit", "MI"],
|
||||
] == response.json["rows"]
|
||||
|
||||
|
||||
def test_sql_time_limit(app_client_shorter_time_limit):
|
||||
response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)")
|
||||
assert 400 == response.status
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from bs4 import BeautifulSoup as Soup
|
||||
import pytest
|
||||
from .fixtures import make_app_client
|
||||
import re
|
||||
from .fixtures import make_app_client, app_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -39,6 +41,18 @@ def canned_write_client():
|
|||
yield client
|
||||
|
||||
|
||||
def test_canned_query_with_named_parameter(app_client):
|
||||
response = app_client.get("/fixtures/neighborhood_search.json?text=town")
|
||||
assert [
|
||||
["Corktown", "Detroit", "MI"],
|
||||
["Downtown", "Los Angeles", "CA"],
|
||||
["Downtown", "Detroit", "MI"],
|
||||
["Greektown", "Detroit", "MI"],
|
||||
["Koreatown", "Los Angeles", "CA"],
|
||||
["Mexicantown", "Detroit", "MI"],
|
||||
] == response.json["rows"]
|
||||
|
||||
|
||||
def test_insert(canned_write_client):
|
||||
response = canned_write_client.post(
|
||||
"/data/add_name", {"name": "Hello"}, allow_redirects=False, csrftoken_from=True,
|
||||
|
|
@ -147,3 +161,74 @@ def test_canned_query_permissions(canned_write_client):
|
|||
cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})}
|
||||
assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status
|
||||
assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def magic_parameters_client():
|
||||
with make_app_client(
|
||||
extra_databases={"data.db": "create table logs (line text)"},
|
||||
metadata={
|
||||
"databases": {
|
||||
"data": {
|
||||
"queries": {
|
||||
"runme_post": {"sql": "", "write": True},
|
||||
"runme_get": {"sql": ""},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"magic_parameter,expected_re",
|
||||
[
|
||||
("_actor_id", "root"),
|
||||
("_header_host", "localhost"),
|
||||
("_cookie_foo", "bar"),
|
||||
("_timestamp_epoch", r"^\d+$"),
|
||||
("_timestamp_date_utc", r"^\d{4}-\d{2}-\d{2}$"),
|
||||
("_timestamp_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"),
|
||||
("_random_chars_1", r"^\w$"),
|
||||
("_random_chars_10", r"^\w{10}$"),
|
||||
],
|
||||
)
|
||||
def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re):
|
||||
magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_post"][
|
||||
"sql"
|
||||
] = "insert into logs (line) values (:{})".format(magic_parameter)
|
||||
magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_get"][
|
||||
"sql"
|
||||
] = "select :{} as result".format(magic_parameter)
|
||||
cookies = {
|
||||
"ds_actor": magic_parameters_client.actor_cookie({"id": "root"}),
|
||||
"foo": "bar",
|
||||
}
|
||||
# Test the GET version
|
||||
get_response = magic_parameters_client.get(
|
||||
"/data/runme_get.json?_shape=array", cookies=cookies
|
||||
)
|
||||
get_actual = get_response.json[0]["result"]
|
||||
assert re.match(expected_re, str(get_actual))
|
||||
# Test the form
|
||||
form_response = magic_parameters_client.get("/data/runme_post")
|
||||
soup = Soup(form_response.body, "html.parser")
|
||||
# The magic parameter should not be represented as a form field
|
||||
assert None is soup.find("input", {"name": magic_parameter})
|
||||
# Submit the form to create a log line
|
||||
response = magic_parameters_client.post(
|
||||
"/data/runme_post", {}, csrftoken_from=True, cookies=cookies
|
||||
)
|
||||
post_actual = magic_parameters_client.get(
|
||||
"/data/logs.json?_sort_desc=rowid&_shape=array"
|
||||
).json[0]["line"]
|
||||
assert re.match(expected_re, post_actual)
|
||||
|
||||
|
||||
def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client):
|
||||
response = magic_parameters_client.get(
|
||||
"/data.json?sql=select+:_header_host&_shape=array"
|
||||
)
|
||||
assert 500 == response.status
|
||||
assert "You did not supply a value for binding 1." == response.json["error"]
|
||||
|
|
@ -638,3 +638,31 @@ def test_canned_queries_actor(app_client):
|
|||
assert [{"1": 1, "actor_id": "bot"}] == app_client.get(
|
||||
"/fixtures/from_hook.json?_bot=1&_shape=array"
|
||||
).json
|
||||
|
||||
|
||||
def test_register_magic_parameters(restore_working_directory):
|
||||
with make_app_client(
|
||||
extra_databases={"data.db": "create table logs (line text)"},
|
||||
metadata={
|
||||
"databases": {
|
||||
"data": {
|
||||
"queries": {
|
||||
"runme": {
|
||||
"sql": "insert into logs (line) values (:_request_http_version)",
|
||||
"write": True,
|
||||
},
|
||||
"get_uuid": {"sql": "select :_uuid_new",},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
) as client:
|
||||
response = client.post("/data/runme", {}, csrftoken_from=True)
|
||||
assert 200 == response.status
|
||||
actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json
|
||||
assert [{"rowid": 1, "line": "1.0"}] == actual
|
||||
# Now try the GET request against get_uuid
|
||||
response_get = client.get("/data/get_uuid.json?_shape=array")
|
||||
assert 200 == response_get.status
|
||||
new_uuid = response_get.json[0][":_uuid_new"]
|
||||
assert 4 == new_uuid.count("-")
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue