Support for generated columns

* Support for generated columns, closes #1116
* Show SQLite version in pytest report header
* Use table_info() if SQLite < 3.26.0
* Cache sqlite_version() rather than re-calculate every time
* Adjust test_database_page for SQLite 3.26.0 or higher

datasette/utils/__init__.py

@@ -19,15 +19,9 @@ import urllib
 import numbers
 import yaml
 from .shutil_backport import copytree
+from .sqlite import sqlite3, sqlite_version
 from ..plugins import pm
 
-try:
-    import pysqlite3 as sqlite3
-except ImportError:
-    import sqlite3
-
-if hasattr(sqlite3, "enable_callback_tracebacks"):
-    sqlite3.enable_callback_tracebacks(True)
 
 # From https://www.sqlite.org/lang_keywords.html
 reserved_words = set(
@@ -64,7 +58,7 @@ HASH_LENGTH = 7
 
 # Can replace this with Column from sqlite_utils when I add that dependency
 Column = namedtuple(
-    "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
+    "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden")
 )
 
 
@@ -458,13 +452,10 @@ def temporary_docker_directory(
 
 def detect_primary_keys(conn, table):
     " Figure out primary keys for a table. "
-    table_info_rows = [
-        row
-        for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall()
-        if row[-1]
-    ]
-    table_info_rows.sort(key=lambda row: row[-1])
-    return [str(r[1]) for r in table_info_rows]
+    columns = table_column_details(conn, table)
+    pks = [column for column in columns if column.is_pk]
+    pks.sort(key=lambda column: column.is_pk)
+    return [column.name for column in pks]
 
 
 def get_outbound_foreign_keys(conn, table):
@@ -570,10 +561,22 @@ def table_columns(conn, table):
 
 
 def table_column_details(conn, table):
-    return [
-        Column(*r)
-        for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall()
-    ]
+    if sqlite_version() >= (3, 26, 0):
+        # table_xinfo was added in 3.26.0
+        return [
+            Column(*r)
+            for r in conn.execute(
+                f"PRAGMA table_xinfo({escape_sqlite(table)});"
+            ).fetchall()
+        ]
+    else:
+        # Treat hidden as 0 for all columns
+        return [
+            Column(*(list(r) + [0]))
+            for r in conn.execute(
+                f"PRAGMA table_info({escape_sqlite(table)});"
+            ).fetchall()
+        ]
 
 
 filter_column_re = re.compile(r"^_filter_column_\d+$")

datasette/utils/sqlite.py

@@ -0,0 +1,28 @@
+try:
+    import pysqlite3 as sqlite3
+except ImportError:
+    import sqlite3
+
+if hasattr(sqlite3, "enable_callback_tracebacks"):
+    sqlite3.enable_callback_tracebacks(True)
+
+_cached_sqlite_version = None
+
+
+def sqlite_version():
+    global _cached_sqlite_version
+    if _cached_sqlite_version is None:
+        _cached_sqlite_version = _sqlite_version()
+    return _cached_sqlite_version
+
+
+def _sqlite_version():
+    return tuple(
+        map(
+            int,
+            sqlite3.connect(":memory:")
+            .execute("select sqlite_version()")
+            .fetchone()[0]
+            .split("."),
+        )
+    )

tests/conftest.py

@@ -3,6 +3,11 @@ import pathlib
 import pytest
 import re
 
+try:
+    import pysqlite3 as sqlite3
+except ImportError:
+    import sqlite3
+
 UNDOCUMENTED_PERMISSIONS = {
     "this_is_allowed",
     "this_is_denied",
@@ -12,6 +17,12 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
+def pytest_report_header(config):
+    return "SQLite: {}".format(
+        sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0]
+    )
+
+
 def pytest_configure(config):
     import sys
 

tests/fixtures.py

@@ -1,5 +1,5 @@
 from datasette.app import Datasette
-from datasette.utils import sqlite3
+from datasette.utils.sqlite import sqlite3
 from datasette.utils.testing import TestClient
 import click
 import contextlib

tests/test_api.py

@@ -1,5 +1,7 @@
+from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
 from datasette.utils import detect_json1
+from datasette.utils.sqlite import sqlite3, sqlite_version
 from datasette.version import __version__
 from .fixtures import (  # noqa
     app_client,
@@ -514,7 +516,20 @@ def test_database_page(app_client):
         },
         {
             "name": "searchable_fts",
-            "columns": ["text1", "text2", "name with . and spaces"],
+            "columns": [
+                "text1",
+                "text2",
+                "name with . and spaces",
+            ]
+            + (
+                [
+                    "searchable_fts",
+                    "docid",
+                    "__langid",
+                ]
+                if sqlite_version() >= (3, 26, 0)
+                else []
+            ),
             "primary_keys": [],
             "count": 2,
             "hidden": True,
@@ -1913,3 +1928,37 @@ def test_paginate_using_link_header(app_client, qs):
         else:
             path = None
     assert num_pages == 21
+
+
+@pytest.mark.skipif(
+    sqlite_version() < (3, 31, 0),
+    reason="generated columns were added in SQLite 3.31.0",
+)
+@pytest.mark.asyncio
+async def test_generated_columns_are_visible_in_datasette(tmp_path_factory):
+    db_directory = tmp_path_factory.mktemp("dbs")
+    db_path = db_directory / "test.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.executescript(
+        """
+    CREATE TABLE deeds (
+        body TEXT,
+        id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED,
+        consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED
+    );
+    INSERT INTO deeds (body) VALUES ('{
+        "id": 1,
+        "consideration": "This is the consideration"
+    }');
+    """
+    )
+    datasette = Datasette([db_path])
+    response = await datasette.client.get("/test/deeds.json?_shape=array")
+    assert response.json() == [
+        {
+            "rowid": 1,
+            "body": '{\n        "id": 1,\n        "consideration": "This is the consideration"\n    }',
+            "id": 1,
+            "consideration": "This is the consideration",
+        }
+    ]

tests/test_config_dir.py

@@ -1,9 +1,9 @@
 import json
 import pytest
-import sqlite3
 
 from datasette.app import Datasette
 from datasette.cli import cli
+from datasette.utils.sqlite import sqlite3
 from .fixtures import TestClient as _TestClient
 from click.testing import CliRunner
 

tests/test_internals_database.py

@@ -2,7 +2,8 @@
 Tests for the datasette.database.Database class
 """
 from datasette.database import Database, Results, MultipleValues
-from datasette.utils import sqlite3, Column
+from datasette.utils.sqlite import sqlite3
+from datasette.utils import Column
 from .fixtures import app_client
 import pytest
 import time
@@ -120,6 +121,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=1,
+                    hidden=0,
                 ),
                 Column(
                     cid=1,
@@ -128,6 +130,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=2,
@@ -136,6 +139,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=3,
@@ -144,6 +148,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=4,
@@ -152,6 +157,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=5,
@@ -160,6 +166,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=6,
@@ -168,6 +175,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=7,
@@ -176,6 +184,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=8,
@@ -184,6 +193,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=9,
@@ -192,6 +202,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
             ],
         ),
@@ -205,6 +216,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=1,
+                    hidden=0,
                 ),
                 Column(
                     cid=1,
@@ -213,6 +225,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=2,
+                    hidden=0,
                 ),
                 Column(
                     cid=2,
@@ -221,6 +234,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=3,
@@ -229,6 +243,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=4,
@@ -237,6 +252,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=5,
@@ -245,6 +261,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
                 Column(
                     cid=6,
@@ -253,6 +270,7 @@ async def test_table_columns(db, table, expected):
                     notnull=0,
                     default_value=None,
                     is_pk=0,
+                    hidden=0,
                 ),
             ],
         ),

tests/test_plugins.py

@@ -9,14 +9,14 @@ from .fixtures import (
 from datasette.app import Datasette
 from datasette import cli
 from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
-from datasette.utils import sqlite3, CustomRow
+from datasette.utils.sqlite import sqlite3
+from datasette.utils import CustomRow
 from jinja2.environment import Template
 import base64
 import json
 import os
 import pathlib
 import re
-import sqlite3
 import textwrap
 import pytest
 import urllib

tests/test_utils.py

@@ -4,11 +4,11 @@ Tests for various datasette helper functions.
 from datasette.app import Datasette
 from datasette import utils
 from datasette.utils.asgi import Request
+from datasette.utils.sqlite import sqlite3
 import json
 import os
 import pathlib
 import pytest
-import sqlite3
 import tempfile
 from unittest.mock import patch