'datasette --get' option, closes #926

Also made a start on the datasette.utils.testing module, refs #898

datasette/cli.py

@@ -21,6 +21,7 @@ from .utils import (
     StaticMount,
     ValueAsBooleanError,
 )
+from .utils.testing import TestClient
 
 
 class Config(click.ParamType):
@@ -335,6 +336,9 @@ def uninstall(packages, yes):
     help="Output URL that sets a cookie authenticating the root user",
     is_flag=True,
 )
+@click.option(
+    "--get", help="Run an HTTP GET request against this path, print results and exit",
+)
 @click.option("--version-note", help="Additional note to show on /-/versions")
 @click.option("--help-config", is_flag=True, help="Show available config options")
 def serve(
@@ -355,6 +359,7 @@ def serve(
     config,
     secret,
     root,
+    get,
     version_note,
     help_config,
     return_instance=False,
@@ -411,6 +416,12 @@ def serve(
 
     ds = Datasette(files, **kwargs)
 
+    if get:
+        client = TestClient(ds.app())
+        response = client.get(get)
+        click.echo(response.text)
+        return
+
     if return_instance:
         # Private utility mechanism for writing unit tests
         return ds

datasette/utils/testing.py

@@ -0,0 +1,151 @@
+from datasette.utils import MultiParams
+from asgiref.testing import ApplicationCommunicator
+from asgiref.sync import async_to_sync
+from urllib.parse import unquote, quote, urlencode
+from http.cookies import SimpleCookie
+import json
+
+
+class TestResponse:
+    def __init__(self, status, headers, body):
+        self.status = status
+        self.headers = headers
+        self.body = body
+
+    @property
+    def cookies(self):
+        cookie = SimpleCookie()
+        for header in self.headers.getlist("set-cookie"):
+            cookie.load(header)
+        return {key: value.value for key, value in cookie.items()}
+
+    @property
+    def json(self):
+        return json.loads(self.text)
+
+    @property
+    def text(self):
+        return self.body.decode("utf8")
+
+
+class TestClient:
+    max_redirects = 5
+
+    def __init__(self, asgi_app):
+        self.asgi_app = asgi_app
+
+    def actor_cookie(self, actor):
+        return self.ds.sign({"a": actor}, "actor")
+
+    @async_to_sync
+    async def get(
+        self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None
+    ):
+        return await self._request(
+            path, allow_redirects, redirect_count, method, cookies
+        )
+
+    @async_to_sync
+    async def post(
+        self,
+        path,
+        post_data=None,
+        allow_redirects=True,
+        redirect_count=0,
+        content_type="application/x-www-form-urlencoded",
+        cookies=None,
+        csrftoken_from=None,
+    ):
+        cookies = cookies or {}
+        post_data = post_data or {}
+        # Maybe fetch a csrftoken first
+        if csrftoken_from is not None:
+            if csrftoken_from is True:
+                csrftoken_from = path
+            token_response = await self._request(csrftoken_from, cookies=cookies)
+            csrftoken = token_response.cookies["ds_csrftoken"]
+            cookies["ds_csrftoken"] = csrftoken
+            post_data["csrftoken"] = csrftoken
+        return await self._request(
+            path,
+            allow_redirects,
+            redirect_count,
+            "POST",
+            cookies,
+            post_data,
+            content_type,
+        )
+
+    async def _request(
+        self,
+        path,
+        allow_redirects=True,
+        redirect_count=0,
+        method="GET",
+        cookies=None,
+        post_data=None,
+        content_type=None,
+    ):
+        query_string = b""
+        if "?" in path:
+            path, _, query_string = path.partition("?")
+            query_string = query_string.encode("utf8")
+        if "%" in path:
+            raw_path = path.encode("latin-1")
+        else:
+            raw_path = quote(path, safe="/:,").encode("latin-1")
+        headers = [[b"host", b"localhost"]]
+        if content_type:
+            headers.append((b"content-type", content_type.encode("utf-8")))
+        if cookies:
+            sc = SimpleCookie()
+            for key, value in cookies.items():
+                sc[key] = value
+            headers.append([b"cookie", sc.output(header="").encode("utf-8")])
+        scope = {
+            "type": "http",
+            "http_version": "1.0",
+            "method": method,
+            "path": unquote(path),
+            "raw_path": raw_path,
+            "query_string": query_string,
+            "headers": headers,
+        }
+        instance = ApplicationCommunicator(self.asgi_app, scope)
+
+        if post_data:
+            body = urlencode(post_data, doseq=True).encode("utf-8")
+            await instance.send_input({"type": "http.request", "body": body})
+        else:
+            await instance.send_input({"type": "http.request"})
+
+        # First message back should be response.start with headers and status
+        messages = []
+        start = await instance.receive_output(2)
+        messages.append(start)
+        assert start["type"] == "http.response.start"
+        response_headers = MultiParams(
+            [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]]
+        )
+        status = start["status"]
+        # Now loop until we run out of response.body
+        body = b""
+        while True:
+            message = await instance.receive_output(2)
+            messages.append(message)
+            assert message["type"] == "http.response.body"
+            body += message["body"]
+            if not message.get("more_body"):
+                break
+        response = TestResponse(status, response_headers, body)
+        if allow_redirects and response.status in (301, 302):
+            assert (
+                redirect_count < self.max_redirects
+            ), "Redirected {} times, max_redirects={}".format(
+                redirect_count, self.max_redirects
+            )
+            location = response.headers["Location"]
+            return await self._request(
+                location, allow_redirects=True, redirect_count=redirect_count + 1
+            )
+        return response

docs/datasette-serve-help.txt

@@ -33,6 +33,9 @@ Options:
                             cookies
 
   --root                    Output URL that sets a cookie authenticating the root user
+  --get TEXT                Run an HTTP GET request against this path, print results and
+                            exit
+
   --version-note TEXT       Additional note to show on /-/versions
   --help-config             Show available config options
   --help                    Show this message and exit.

docs/getting_started.rst

@@ -9,7 +9,7 @@ The best way to experience Datasette for the first time is with a demo:
 * `fivethirtyeight.datasettes.com <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository <https://github.com/fivethirtyeight/data>`__.
 * `sf-trees.datasettes.com <https://sf-trees.datasettes.com/trees/Street_Tree_List>`__ demonstrates the `datasette-cluster-map <https://github.com/simonw/datasette-cluster-map>`__ plugin running against 190,000 trees imported from `data.sfgov.org <https://data.sfgov.org/City-Infrastructure/Street-Tree-List/tkzw-k3nq>`__.
 
-.. _glitch:
+.. _getting_started_glitch:
 
 Try Datasette without installing anything using Glitch
 ------------------------------------------------------
@@ -33,6 +33,8 @@ Need some data? Try this `Public Art Data <https://data.seattle.gov/Community/Pu
 
 For more on how this works, see `Running Datasette on Glitch <https://simonwillison.net/2019/Apr/23/datasette-glitch/>`__.
 
+.. _getting_started_your_computer:
+
 Using Datasette on your own computer
 ------------------------------------
 
@@ -40,13 +42,11 @@ First, follow the :ref:`installation` instructions. Now you can run Datasette ag
 
 ::
 
-    datasette serve path/to/database.db
+    datasette path/to/database.db
 
 This will start a web server on port 8001 - visit http://localhost:8001/
 to access the web interface.
 
-``serve`` is the default subcommand, you can omit it if you like.
-
 Use Chrome on OS X? You can run datasette against your browser history
 like so:
 
@@ -90,7 +90,7 @@ JSON:
     }
 
 http://localhost:8001/History/downloads.json?_shape=objects will return that data as
-JSON in a more convenient but less efficient format:
+JSON in a more convenient format:
 
 ::
 
@@ -109,7 +109,57 @@ JSON in a more convenient but less efficient format:
         ]
     }
 
-datasette serve options
+.. _getting_started_datasette_get:
------------------------
+
+datasette --get
+---------------
+
+The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example::
+
+    $ datasette --get '/-/versions.json' | jq .
+    {
+      "python": {
+        "version": "3.8.5",
+        "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]"
+      },
+      "datasette": {
+        "version": "0.46+15.g222a84a.dirty"
+      },
+      "asgi": "3.0",
+      "uvicorn": "0.11.8",
+      "sqlite": {
+        "version": "3.32.3",
+        "fts_versions": [
+          "FTS5",
+          "FTS4",
+          "FTS3"
+        ],
+        "extensions": {
+          "json1": null
+        },
+        "compile_options": [
+          "COMPILER=clang-11.0.3",
+          "ENABLE_COLUMN_METADATA",
+          "ENABLE_FTS3",
+          "ENABLE_FTS3_PARENTHESIS",
+          "ENABLE_FTS4",
+          "ENABLE_FTS5",
+          "ENABLE_GEOPOLY",
+          "ENABLE_JSON1",
+          "ENABLE_PREUPDATE_HOOK",
+          "ENABLE_RTREE",
+          "ENABLE_SESSION",
+          "MAX_VARIABLE_NUMBER=250000",
+          "THREADSAFE=1"
+        ]
+      }
+    }
+
+.. _getting_started_serve_help:
+
+datasette serve --help
+----------------------
+
+Running ``datasette downloads.db`` executes the default ``serve`` sub-command, and is equivalent to running ``datasette serve downloads.db``. The full list of options to that command is shown below.
 
 .. literalinclude:: datasette-serve-help.txt

docs/index.rst

@@ -23,7 +23,7 @@ Datasette is a tool for exploring and publishing data. It helps people take data
 
 Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. It is part of a :ref:`wider ecosystem of tools and plugins <ecosystem>` dedicated to making working with structured data as productive as possible.
 
-`Explore a demo <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__, watch `a presentation about the project <https://static.simonwillison.net/static/2018/pybay-datasette/>`__ or :ref:`glitch`.
+`Explore a demo <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__, watch `a presentation about the project <https://static.simonwillison.net/static/2018/pybay-datasette/>`__ or :ref:`getting_started_glitch`.
 
 More examples: https://github.com/simonw/datasette/wiki/Datasettes
 

docs/installation.rst

@@ -5,7 +5,7 @@
 ==============
 
 .. note::
-    If you just want to try Datasette out you don't need to install anything: see :ref:`glitch`
+    If you just want to try Datasette out you don't need to install anything: see :ref:`getting_started_glitch`
 
 There are two main options for installing Datasette. You can install it directly
 on to your machine, or you can install it using Docker.

setup.py

@@ -44,6 +44,7 @@ setup(
     package_data={"datasette": ["templates/*.html"]},
     include_package_data=True,
     install_requires=[
+        "asgiref~=3.2.10",
         "click~=7.1.1",
         "click-default-group~=1.2.2",
         "Jinja2>=2.10.3,<2.12.0",
@@ -70,7 +71,6 @@ setup(
             "pytest>=5.2.2,<6.1.0",
             "pytest-asyncio>=0.10,<0.15",
             "beautifulsoup4>=4.8.1,<4.10.0",
-            "asgiref~=3.2.3",
             "black~=19.10b0",
         ],
     },

tests/fixtures.py

@@ -1,10 +1,8 @@
 from datasette.app import Datasette
-from datasette.utils import sqlite3, MultiParams
+from datasette.utils import sqlite3
-from asgiref.testing import ApplicationCommunicator
+from datasette.utils.testing import TestClient
-from asgiref.sync import async_to_sync
 import click
 import contextlib
-from http.cookies import SimpleCookie
 import itertools
 import json
 import os
@@ -16,7 +14,6 @@ import string
 import tempfile
 import textwrap
 import time
-from urllib.parse import unquote, quote, urlencode
 
 
 # This temp file is used by one of the plugin config tests
@@ -89,151 +86,6 @@ EXPECTED_PLUGINS = [
 ]
 
 
-class TestResponse:
-    def __init__(self, status, headers, body):
-        self.status = status
-        self.headers = headers
-        self.body = body
-
-    @property
-    def cookies(self):
-        cookie = SimpleCookie()
-        for header in self.headers.getlist("set-cookie"):
-            cookie.load(header)
-        return {key: value.value for key, value in cookie.items()}
-
-    @property
-    def json(self):
-        return json.loads(self.text)
-
-    @property
-    def text(self):
-        return self.body.decode("utf8")
-
-
-class TestClient:
-    max_redirects = 5
-
-    def __init__(self, asgi_app):
-        self.asgi_app = asgi_app
-
-    def actor_cookie(self, actor):
-        return self.ds.sign({"a": actor}, "actor")
-
-    @async_to_sync
-    async def get(
-        self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None
-    ):
-        return await self._request(
-            path, allow_redirects, redirect_count, method, cookies
-        )
-
-    @async_to_sync
-    async def post(
-        self,
-        path,
-        post_data=None,
-        allow_redirects=True,
-        redirect_count=0,
-        content_type="application/x-www-form-urlencoded",
-        cookies=None,
-        csrftoken_from=None,
-    ):
-        cookies = cookies or {}
-        post_data = post_data or {}
-        # Maybe fetch a csrftoken first
-        if csrftoken_from is not None:
-            if csrftoken_from is True:
-                csrftoken_from = path
-            token_response = await self._request(csrftoken_from, cookies=cookies)
-            csrftoken = token_response.cookies["ds_csrftoken"]
-            cookies["ds_csrftoken"] = csrftoken
-            post_data["csrftoken"] = csrftoken
-        return await self._request(
-            path,
-            allow_redirects,
-            redirect_count,
-            "POST",
-            cookies,
-            post_data,
-            content_type,
-        )
-
-    async def _request(
-        self,
-        path,
-        allow_redirects=True,
-        redirect_count=0,
-        method="GET",
-        cookies=None,
-        post_data=None,
-        content_type=None,
-    ):
-        query_string = b""
-        if "?" in path:
-            path, _, query_string = path.partition("?")
-            query_string = query_string.encode("utf8")
-        if "%" in path:
-            raw_path = path.encode("latin-1")
-        else:
-            raw_path = quote(path, safe="/:,").encode("latin-1")
-        headers = [[b"host", b"localhost"]]
-        if content_type:
-            headers.append((b"content-type", content_type.encode("utf-8")))
-        if cookies:
-            sc = SimpleCookie()
-            for key, value in cookies.items():
-                sc[key] = value
-            headers.append([b"cookie", sc.output(header="").encode("utf-8")])
-        scope = {
-            "type": "http",
-            "http_version": "1.0",
-            "method": method,
-            "path": unquote(path),
-            "raw_path": raw_path,
-            "query_string": query_string,
-            "headers": headers,
-        }
-        instance = ApplicationCommunicator(self.asgi_app, scope)
-
-        if post_data:
-            body = urlencode(post_data, doseq=True).encode("utf-8")
-            await instance.send_input({"type": "http.request", "body": body})
-        else:
-            await instance.send_input({"type": "http.request"})
-
-        # First message back should be response.start with headers and status
-        messages = []
-        start = await instance.receive_output(2)
-        messages.append(start)
-        assert start["type"] == "http.response.start"
-        response_headers = MultiParams(
-            [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]]
-        )
-        status = start["status"]
-        # Now loop until we run out of response.body
-        body = b""
-        while True:
-            message = await instance.receive_output(2)
-            messages.append(message)
-            assert message["type"] == "http.response.body"
-            body += message["body"]
-            if not message.get("more_body"):
-                break
-        response = TestResponse(status, response_headers, body)
-        if allow_redirects and response.status in (301, 302):
-            assert (
-                redirect_count < self.max_redirects
-            ), "Redirected {} times, max_redirects={}".format(
-                redirect_count, self.max_redirects
-            )
-            location = response.headers["Location"]
-            return await self._request(
-                location, allow_redirects=True, redirect_count=redirect_count + 1
-            )
-        return response
-
-
 @contextlib.contextmanager
 def make_app_client(
     sql_time_limit_ms=None,

tests/test_cli.py

@@ -50,6 +50,20 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache():
         assert {"hithere": 44} == db.cached_table_counts
 
 
+def test_serve_with_get():
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        ["serve", "--memory", "--get", "/:memory:.json?sql=select+sqlite_version()"],
+    )
+    assert 0 == result.exit_code, result.output
+    assert {
+        "database": ":memory:",
+        "truncated": False,
+        "columns": ["sqlite_version()"],
+    }.items() <= json.loads(result.output).items()
+
+
 def test_spatialite_error_if_attempt_to_open_spatialite():
     runner = CliRunner()
     result = runner.invoke(
@@ -102,6 +116,7 @@ def test_metadata_yaml():
         secret=None,
         root=False,
         version_note=None,
+        get=None,
         help_config=False,
         return_instance=True,
     )