From 187d91d68617ca48e34c1fb0c6722a40f8567d45 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Nov 2022 21:57:28 -0800 Subject: [PATCH] /db/-/create API endpoint, closes #1882 --- datasette/app.py | 3 +- datasette/default_permissions.py | 1 + datasette/views/database.py | 133 ++++++++++++- docs/authentication.rst | 12 ++ docs/json_api.rst | 111 ++++++++++- tests/test_api_write.py | 315 +++++++++++++++++++++++++++++++ 6 files changed, 572 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 02bd38f1..2678356a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -27,7 +27,7 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView +from .views.database import DatabaseDownload, DatabaseView, TableCreateView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1265,6 +1265,7 @@ class Datasette: add_route( DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" ) + add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( TableView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 0964d536..daaec8a7 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,6 +13,7 @@ def permission_allowed_default(datasette, actor, action, resource): "permissions-debug", "debug-menu", "insert-row", + "create-table", "drop-table", "delete-row", ): diff --git a/datasette/views/database.py b/datasette/views/database.py index 8e08c3b1..a306a666 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -4,6 +4,8 @@ import itertools import json from markupsafe import Markup, escape from urllib.parse import parse_qsl, urlencode +import re +import sqlite_utils import markupsafe @@ -26,7 +28,7 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import DatasetteError, DataView +from .base import BaseView, DatasetteError, DataView, _error class DatabaseView(DataView): @@ -554,3 +556,132 @@ class MagicParameters(dict): return super().__getitem__(key) else: return super().__getitem__(key) + + +class TableCreateView(BaseView): + name = "table-create" + + _valid_keys = {"table", "rows", "row", "columns", "pk"} + _supported_column_types = { + "text", + "integer", + "float", + "blob", + } + # Any string that does not contain a newline or start with sqlite_ + _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + return _error(["Database not found: {}".format(database_route)], 404) + database_name = db.name + + # Must have create-table permission + if not await self.ds.permission_allowed( + request.actor, "create-table", resource=database_name + ): + return _error(["Permission denied"], 403) + + body = await request.post_body() + try: + data = json.loads(body) + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)]) + + if not isinstance(data, dict): + return _error(["JSON must be an object"]) + + invalid_keys = set(data.keys()) - self._valid_keys + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + + table_name = data.get("table") + if not table_name: + return _error(["Table is required"]) + + if not self._table_name_re.match(table_name): + return _error(["Invalid table name"]) + + columns = data.get("columns") + rows = data.get("rows") + row = data.get("row") + if not columns and not rows and not row: + return _error(["columns, rows or row is required"]) + + if rows and row: + return _error(["Cannot specify both rows and row"]) + + if columns: + if rows or row: + return _error(["Cannot specify columns with rows or row"]) + if not isinstance(columns, list): + return _error(["columns must be a list"]) + for column in columns: + if not isinstance(column, dict): + return _error(["columns must be a list of objects"]) + if not column.get("name") or not isinstance(column.get("name"), str): + return _error(["Column name is required"]) + if not column.get("type"): + column["type"] = "text" + if column["type"] not in self._supported_column_types: + return _error( + ["Unsupported column type: {}".format(column["type"])] + ) + # No duplicate column names + dupes = {c["name"] for c in columns if columns.count(c) > 1} + if dupes: + return _error(["Duplicate column name: {}".format(", ".join(dupes))]) + + if row: + rows = [row] + + if rows: + if not isinstance(rows, list): + return _error(["rows must be a list"]) + for row in rows: + if not isinstance(row, dict): + return _error(["rows must be a list of objects"]) + + pk = data.get("pk") + if pk: + if not isinstance(pk, str): + return _error(["pk must be a string"]) + + def create_table(conn): + table = sqlite_utils.Database(conn)[table_name] + if rows: + table.insert_all(rows, pk=pk) + else: + table.create( + {c["name"]: c["type"] for c in columns}, + pk=pk, + ) + return table.schema + + try: + schema = await db.execute_write_fn(create_table) + except Exception as e: + return _error([str(e)]) + table_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name, format="json") + ) + details = { + "ok": True, + "database": db.name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "schema": schema, + } + if rows: + details["row_count"] = len(rows) + return Response.json(details, status=201) diff --git a/docs/authentication.rst b/docs/authentication.rst index f59ac520..a86c82a2 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -589,6 +589,18 @@ Actor is allowed to delete rows from a table. Default *deny*. +.. _permissions_create_table: + +create-table +------------ + +Actor is allowed to create a database table. + +``resource`` - string + The name of the database + +Default *deny*. + .. _permissions_drop_table: drop-table diff --git a/docs/json_api.rst b/docs/json_api.rst index 842285cf..ffdbf8d3 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -415,7 +415,9 @@ column - you can turn that off using ``?_labels=off``. You can request foreign keys be expanded in JSON using the ``_labels=on`` or ``_label=COLUMN`` special query string parameters. Here's what an expanded row -looks like:: +looks like: + +.. code-block:: json [ { @@ -477,6 +479,9 @@ A single row can be inserted using the ``"row"`` key: POST //
/-/insert Content-Type: application/json Authorization: Bearer dstok_ + +.. code-block:: json + { "row": { "column1": "value1", @@ -505,6 +510,9 @@ To insert multiple rows at a time, use the same API method but send a list of di POST //
/-/insert Content-Type: application/json Authorization: Bearer dstok_ + +.. code-block:: json + { "rows": [ { @@ -559,6 +567,107 @@ If successful, this will return a ``200`` status code and a ``{"ok": true}`` res Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +.. _TableCreateView: + +Creating a table +~~~~~~~~~~~~~~~~ + +To create a table, make a ``POST`` to ``//-/create``. This requires the :ref:`permissions_create_table` permission. + +:: + + POST //-/create + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "table": "name_of_new_table", + "columns": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "title", + "type": "text" + } + ], + "pk": "id" + } + +The JSON here describes the table that will be created: + +* ``table`` is the name of the table to create. This field is required. +* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. + + - ``name`` is the name of the column. This is required. + - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + +* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. + + If the primary key is an integer column, it will be configured to automatically increment for each new record. + + If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an integer ID column for you. + +If the table is successfully created this will return a ``201`` status code and the following response: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "name_of_new_table", + "table_url": "http://127.0.0.1:8001/data/name_of_new_table", + "table_api_url": "http://127.0.0.1:8001/data/name_of_new_table.json", + "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT\n)" + } + +.. _TableCreateView_example: + +Creating a table from example data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Instead of specifying ``columns`` directly you can instead pass a single example row or a list of rows. Datasette will create a table with a schema that matches those rows and insert them for you: + +:: + + POST //-/create + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "table": "creatures", + "rows": [ + { + "id": 1, + "name": "Tarantula" + }, + { + "id": 2, + "name": "Kākāpō" + } + ], + "pk": "id" + } + +The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "creatures", + "table_url": "http://127.0.0.1:8001/data/creatures", + "table_api_url": "http://127.0.0.1:8001/data/creatures.json", + "schema": "CREATE TABLE [creatures] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + "row_count": 2 + } + .. _TableDropView: Dropping tables diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 5568279c..f455775b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -405,3 +405,318 @@ async def test_drop_table(ds_write, scenario): ) assert response2.json() == {"ok": True} assert (await ds_write.client.get("/data/docs")).status_code == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "input,expected_status,expected_response", + ( + # Permission error with a bad token + ( + {"table": "bad", "row": {"id": 1}}, + 403, + {"ok": False, "errors": ["Permission denied"]}, + ), + # Successful creation with columns: + ( + { + "table": "one", + "columns": [ + { + "name": "id", + "type": "integer", + }, + { + "name": "title", + "type": "text", + }, + { + "name": "score", + "type": "integer", + }, + { + "name": "weight", + "type": "float", + }, + { + "name": "thumbnail", + "type": "blob", + }, + ], + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "one", + "table_url": "http://localhost/data/one", + "table_api_url": "http://localhost/data/one.json", + "schema": ( + "CREATE TABLE [one] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] INTEGER,\n" + " [weight] FLOAT,\n" + " [thumbnail] BLOB\n" + ")" + ), + }, + ), + # Successful creation with rows: + ( + { + "table": "two", + "rows": [ + { + "id": 1, + "title": "Row 1", + "score": 1.5, + }, + { + "id": 2, + "title": "Row 2", + "score": 1.5, + }, + ], + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "two", + "table_url": "http://localhost/data/two", + "table_api_url": "http://localhost/data/two.json", + "schema": ( + "CREATE TABLE [two] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" + ")" + ), + "row_count": 2, + }, + ), + # Successful creation with row: + ( + { + "table": "three", + "row": { + "id": 1, + "title": "Row 1", + "score": 1.5, + }, + "pk": "id", + }, + 201, + { + "ok": True, + "database": "data", + "table": "three", + "table_url": "http://localhost/data/three", + "table_api_url": "http://localhost/data/three.json", + "schema": ( + "CREATE TABLE [three] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [score] FLOAT\n" + ")" + ), + "row_count": 1, + }, + ), + # Create with row and no primary key + ( + { + "table": "four", + "row": { + "name": "Row 1", + }, + }, + 201, + { + "ok": True, + "database": "data", + "table": "four", + "table_url": "http://localhost/data/four", + "table_api_url": "http://localhost/data/four.json", + "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), + "row_count": 1, + }, + ), + # Error: Table is required + ( + { + "row": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["Table is required"], + }, + ), + # Error: Invalid table name + ( + { + "table": "sqlite_bad_name", + "row": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["Invalid table name"], + }, + ), + # Error: JSON must be an object + ( + [], + 400, + { + "ok": False, + "errors": ["JSON must be an object"], + }, + ), + # Error: Cannot specify columns with rows or row + ( + { + "table": "bad", + "columns": [{"name": "id", "type": "integer"}], + "rows": [{"id": 1}], + }, + 400, + { + "ok": False, + "errors": ["Cannot specify columns with rows or row"], + }, + ), + # Error: columns, rows or row is required + ( + { + "table": "bad", + }, + 400, + { + "ok": False, + "errors": ["columns, rows or row is required"], + }, + ), + # Error: columns must be a list + ( + { + "table": "bad", + "columns": {"name": "id", "type": "integer"}, + }, + 400, + { + "ok": False, + "errors": ["columns must be a list"], + }, + ), + # Error: columns must be a list of objects + ( + { + "table": "bad", + "columns": ["id"], + }, + 400, + { + "ok": False, + "errors": ["columns must be a list of objects"], + }, + ), + # Error: Column name is required + ( + { + "table": "bad", + "columns": [{"type": "integer"}], + }, + 400, + { + "ok": False, + "errors": ["Column name is required"], + }, + ), + # Error: Unsupported column type + ( + { + "table": "bad", + "columns": [{"name": "id", "type": "bad"}], + }, + 400, + { + "ok": False, + "errors": ["Unsupported column type: bad"], + }, + ), + # Error: Duplicate column name + ( + { + "table": "bad", + "columns": [ + {"name": "id", "type": "integer"}, + {"name": "id", "type": "integer"}, + ], + }, + 400, + { + "ok": False, + "errors": ["Duplicate column name: id"], + }, + ), + # Error: rows must be a list + ( + { + "table": "bad", + "rows": {"id": 1}, + }, + 400, + { + "ok": False, + "errors": ["rows must be a list"], + }, + ), + # Error: rows must be a list of objects + ( + { + "table": "bad", + "rows": ["id"], + }, + 400, + { + "ok": False, + "errors": ["rows must be a list of objects"], + }, + ), + # Error: pk must be a string + ( + { + "table": "bad", + "row": {"id": 1}, + "pk": 1, + }, + 400, + { + "ok": False, + "errors": ["pk must be a string"], + }, + ), + ), +) +async def test_create_table(ds_write, input, expected_status, expected_response): + # Special case for expected status of 403 + if expected_status == 403: + token = "bad_token" + else: + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == expected_status + data = response.json() + assert data == expected_response