From 461670a0b87efa953141b449a9a261919864ceb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 13:29:57 -0800 Subject: [PATCH] Support for generated columns * Support for generated columns, closes #1116 * Show SQLite version in pytest report header * Use table_info() if SQLite < 3.26.0 * Cache sqlite_version() rather than re-calculate every time * Adjust test_database_page for SQLite 3.26.0 or higher --- datasette/utils/__init__.py | 41 +++++++++++++------------ datasette/utils/sqlite.py | 28 ++++++++++++++++++ tests/conftest.py | 11 +++++++ tests/fixtures.py | 2 +- tests/test_api.py | 51 +++++++++++++++++++++++++++++++- tests/test_config_dir.py | 2 +- tests/test_internals_database.py | 20 ++++++++++++- tests/test_plugins.py | 4 +-- tests/test_utils.py | 2 +- 9 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 datasette/utils/sqlite.py diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d467383d..b951539d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -19,15 +19,9 @@ import urllib import numbers import yaml from .shutil_backport import copytree +from .sqlite import sqlite3, sqlite_version from ..plugins import pm -try: - import pysqlite3 as sqlite3 -except ImportError: - import sqlite3 - -if hasattr(sqlite3, "enable_callback_tracebacks"): - sqlite3.enable_callback_tracebacks(True) # From https://www.sqlite.org/lang_keywords.html reserved_words = set( @@ -64,7 +58,7 @@ HASH_LENGTH = 7 # Can replace this with Column from sqlite_utils when I add that dependency Column = namedtuple( - "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") + "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") ) @@ -458,13 +452,10 @@ def temporary_docker_directory( def detect_primary_keys(conn, table): " Figure out primary keys for a table. " - table_info_rows = [ - row - for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() - if row[-1] - ] - table_info_rows.sort(key=lambda row: row[-1]) - return [str(r[1]) for r in table_info_rows] + columns = table_column_details(conn, table) + pks = [column for column in columns if column.is_pk] + pks.sort(key=lambda column: column.is_pk) + return [column.name for column in pks] def get_outbound_foreign_keys(conn, table): @@ -570,10 +561,22 @@ def table_columns(conn, table): def table_column_details(conn, table): - return [ - Column(*r) - for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall() - ] + if sqlite_version() >= (3, 26, 0): + # table_xinfo was added in 3.26.0 + return [ + Column(*r) + for r in conn.execute( + f"PRAGMA table_xinfo({escape_sqlite(table)});" + ).fetchall() + ] + else: + # Treat hidden as 0 for all columns + return [ + Column(*(list(r) + [0])) + for r in conn.execute( + f"PRAGMA table_info({escape_sqlite(table)});" + ).fetchall() + ] filter_column_re = re.compile(r"^_filter_column_\d+$") diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py new file mode 100644 index 00000000..9a043ccd --- /dev/null +++ b/datasette/utils/sqlite.py @@ -0,0 +1,28 @@ +try: + import pysqlite3 as sqlite3 +except ImportError: + import sqlite3 + +if hasattr(sqlite3, "enable_callback_tracebacks"): + sqlite3.enable_callback_tracebacks(True) + +_cached_sqlite_version = None + + +def sqlite_version(): + global _cached_sqlite_version + if _cached_sqlite_version is None: + _cached_sqlite_version = _sqlite_version() + return _cached_sqlite_version + + +def _sqlite_version(): + return tuple( + map( + int, + sqlite3.connect(":memory:") + .execute("select sqlite_version()") + .fetchone()[0] + .split("."), + ) + ) diff --git a/tests/conftest.py b/tests/conftest.py index 91b811e2..a963a4fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,11 @@ import pathlib import pytest import re +try: + import pysqlite3 as sqlite3 +except ImportError: + import sqlite3 + UNDOCUMENTED_PERMISSIONS = { "this_is_allowed", "this_is_denied", @@ -12,6 +17,12 @@ UNDOCUMENTED_PERMISSIONS = { } +def pytest_report_header(config): + return "SQLite: {}".format( + sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] + ) + + def pytest_configure(config): import sys diff --git a/tests/fixtures.py b/tests/fixtures.py index f95a2d6b..b0c98f39 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils import sqlite3 +from datasette.utils.sqlite import sqlite3 from datasette.utils.testing import TestClient import click import contextlib diff --git a/tests/test_api.py b/tests/test_api.py index 848daf9c..5676622e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,7 @@ +from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 +from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -514,7 +516,20 @@ def test_database_page(app_client): }, { "name": "searchable_fts", - "columns": ["text1", "text2", "name with . and spaces"], + "columns": [ + "text1", + "text2", + "name with . and spaces", + ] + + ( + [ + "searchable_fts", + "docid", + "__langid", + ] + if sqlite_version() >= (3, 26, 0) + else [] + ), "primary_keys": [], "count": 2, "hidden": True, @@ -1913,3 +1928,37 @@ def test_paginate_using_link_header(app_client, qs): else: path = None assert num_pages == 21 + + +@pytest.mark.skipif( + sqlite_version() < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +@pytest.mark.asyncio +async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = db_directory / "test.db" + conn = sqlite3.connect(str(db_path)) + conn.executescript( + """ + CREATE TABLE deeds ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED + ); + INSERT INTO deeds (body) VALUES ('{ + "id": 1, + "consideration": "This is the consideration" + }'); + """ + ) + datasette = Datasette([db_path]) + response = await datasette.client.get("/test/deeds.json?_shape=array") + assert response.json() == [ + { + "rowid": 1, + "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', + "id": 1, + "consideration": "This is the consideration", + } + ] diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index cd158474..015c6ace 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,9 +1,9 @@ import json import pytest -import sqlite3 from datasette.app import Datasette from datasette.cli import cli +from datasette.utils.sqlite import sqlite3 from .fixtures import TestClient as _TestClient from click.testing import CliRunner diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 7c8f478c..e50cf20e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,7 +2,8 @@ Tests for the datasette.database.Database class """ from datasette.database import Database, Results, MultipleValues -from datasette.utils import sqlite3, Column +from datasette.utils.sqlite import sqlite3 +from datasette.utils import Column from .fixtures import app_client import pytest import time @@ -120,6 +121,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -128,6 +130,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=2, @@ -136,6 +139,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -144,6 +148,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -152,6 +157,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -160,6 +166,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -168,6 +175,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=7, @@ -176,6 +184,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=8, @@ -184,6 +193,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=9, @@ -192,6 +202,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), @@ -205,6 +216,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -213,6 +225,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=2, + hidden=0, ), Column( cid=2, @@ -221,6 +234,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -229,6 +243,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -237,6 +252,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -245,6 +261,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -253,6 +270,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 51faeccb..4554cfd4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,14 +9,14 @@ from .fixtures import ( from datasette.app import Datasette from datasette import cli from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm -from datasette.utils import sqlite3, CustomRow +from datasette.utils.sqlite import sqlite3 +from datasette.utils import CustomRow from jinja2.environment import Template import base64 import json import os import pathlib import re -import sqlite3 import textwrap import pytest import urllib diff --git a/tests/test_utils.py b/tests/test_utils.py index 07e6f870..56306339 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,11 +4,11 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request +from datasette.utils.sqlite import sqlite3 import json import os import pathlib import pytest -import sqlite3 import tempfile from unittest.mock import patch