diff --git a/datasette/views/table.py b/datasette/views/table.py index 50d2b3c2..fcbe253d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,7 +8,12 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted -from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent +from datasette.events import ( + AlterTableEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, +) from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -388,7 +393,7 @@ class TableInsertView(BaseView): extras = { key: value for key, value in data.items() if key not in ("row", "rows") } - valid_extras = {"return", "ignore", "replace"} + valid_extras = {"return", "ignore", "replace", "alter"} invalid_extras = extras.keys() - valid_extras if invalid_extras: return _errors( @@ -397,7 +402,6 @@ class TableInsertView(BaseView): if extras.get("ignore") and extras.get("replace"): return _errors(['Cannot use "ignore" and "replace" at the same time']) - # Validate columns of each row columns = set(await db.table_columns(table_name)) columns.update(pks_list) @@ -412,7 +416,7 @@ class TableInsertView(BaseView): ) ) invalid_columns = set(row.keys()) - columns - if invalid_columns: + if invalid_columns and not extras.get("alter"): errors.append( "Row {} has invalid columns: {}".format( i, ", ".join(sorted(invalid_columns)) @@ -476,10 +480,23 @@ class TableInsertView(BaseView): ignore = extras.get("ignore") replace = extras.get("replace") + alter = extras.get("alter") if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + initial_schema = None + if alter: + # Must have alter-table permission + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(database_name, table_name) + ): + return _error(["Permission denied for alter-table"], 403) + # Track initial schema to check if it changed later + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + should_return = bool(extras.get("return", False)) row_pk_values_for_later = [] if should_return and upsert: @@ -489,9 +506,13 @@ class TableInsertView(BaseView): table = sqlite_utils.Database(conn)[table_name] kwargs = {} if upsert: - kwargs["pk"] = pks[0] if len(pks) == 1 else pks + kwargs = { + "pk": pks[0] if len(pks) == 1 else pks, + "alter": alter, + } else: - kwargs = {"ignore": ignore, "replace": replace} + # Insert + kwargs = {"ignore": ignore, "replace": replace, "alter": alter} if should_return and not upsert: rowids = [] method = table.upsert if upsert else table.insert @@ -552,6 +573,21 @@ class TableInsertView(BaseView): ) ) + if initial_schema is not None: + after_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + if initial_schema != after_schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=after_schema, + ) + ) + return Response.json(result, status=200 if upsert else 201) diff --git a/docs/json_api.rst b/docs/json_api.rst index 68a0c984..000f532d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -618,6 +618,8 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _TableUpsertView: Upserting rows @@ -728,6 +730,8 @@ When using upsert you must provide the primary key column (or columns if the tab If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowUpdateView: Updating a row @@ -849,7 +853,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. -* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index abf9a88a..9e1d73e0 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -60,6 +60,27 @@ async def test_insert_row(ds_write): assert not event.replace +@pytest.mark.asyncio +async def test_insert_row_alter(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={ + "row": {"title": "Test", "score": 1.2, "age": 5, "extra": "extra"}, + "alter": True, + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert response.json()["ok"] is True + assert response.json()["rows"][0]["extra"] == "extra" + # Analytics event + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema + + @pytest.mark.asyncio @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): @@ -278,16 +299,27 @@ async def test_insert_rows(ds_write, return_rows): 403, ["Permission denied: need both insert-row and update-row"], ), + # Alter table forbidden without alter permission + ( + "/data/docs/-/upsert", + {"rows": [{"id": 1, "title": "One", "extra": "extra"}], "alter": True}, + "update-and-insert-but-no-alter", + 403, + ["Permission denied for alter-table"], + ), ), ) async def test_insert_or_upsert_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): - token = write_token(ds_write) + token_permissions = [] if special_case == "insert-but-not-update": - token = write_token(ds_write, permissions=["ir", "vi"]) + token_permissions = ["ir", "vi"] if special_case == "update-but-not-insert": - token = write_token(ds_write, permissions=["ur", "vi"]) + token_permissions = ["ur", "vi"] + if special_case == "update-and-insert-but-no-alter": + token_permissions = ["ur", "ir"] + token = write_token(ds_write, permissions=token_permissions) if special_case == "duplicate_id": await ds_write.get_database("data").execute_write( "insert into docs (id) values (1)" @@ -309,7 +341,9 @@ async def test_insert_or_upsert_row_errors( actor_response = ( await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) ).json() - print(actor_response) + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -434,6 +468,12 @@ async def test_insert_ignore_replace( {"id": 1, "title": "Two", "score": 1}, ], ), + ( + # Upsert with an alter + {"rows": [{"id": 1, "title": "One"}], "pk": "id"}, + {"rows": [{"id": 1, "title": "Two", "extra": "extra"}], "alter": True}, + [{"id": 1, "title": "Two", "extra": "extra"}], + ), ), ) @pytest.mark.parametrize("should_return", (False, True))