From 249fcf8e3e2a90e763f41b080c1b9ec8017f5005 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:36:10 -0700 Subject: [PATCH 1/9] Add setuptools to dependencies Refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d41e428a..d549280a 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( "itsdangerous>=1.1", "sqlite-utils>=3.30", "asyncinject>=0.5", + "setuptools", ], entry_points=""" [console_scripts] From 0b0c5cd7a94fe3f151a3e10261b5c84ee64f2f18 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 21:20:38 -0700 Subject: [PATCH 2/9] Hopeful fix for Python 3.7 httpx failure, refs #2066 --- tests/test_csv.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index c43e528b..ed83d685 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -6,6 +6,7 @@ from .fixtures import ( # noqa app_client_with_cors, app_client_with_trace, ) +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello @@ -154,11 +155,24 @@ async def test_csv_with_non_ascii_characters(ds_client): def test_max_csv_mb(app_client_csv_max_mb_one): + # This query deliberately generates a really long string + # should be 100*100*100*2 = roughly 2MB response = app_client_csv_max_mb_one.get( - ( - "/fixtures.csv?sql=select+'{}'+" - "from+compound_three_primary_keys&_stream=1&_size=max" - ).format("abcdefg" * 10000) + "/fixtures.csv?" + + urllib.parse.urlencode( + { + "sql": """ + select group_concat('ab', '') + from json_each(json_array({lots})), + json_each(json_array({lots})), + json_each(json_array({lots})) + """.format( + lots=", ".join(str(i) for i in range(100)) + ), + "_stream": 1, + "_size": "max", + } + ), ) # It's a 200 because we started streaming before we knew the error assert response.status == 200 From 55c526a5373aa41c76a3f052624f92e7add59cc8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:35 -0700 Subject: [PATCH 3/9] Add pip as a dependency too, for Rye - refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d549280a..d6824255 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( "sqlite-utils>=3.30", "asyncinject>=0.5", "setuptools", + "pip", ], entry_points=""" [console_scripts] From d3d16b5ccfe59e069113699838c8bf0956d90661 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:44:27 -0700 Subject: [PATCH 4/9] Build docs with 3.11 on ReadTheDocs Inspired by https://github.com/simonw/sqlite-utils/issues/540 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e157fb9c..5b30e75a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py From 49184c569cd70efbda4f3f062afef3a34401d8d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 May 2023 09:24:28 -0700 Subject: [PATCH 5/9] Action: Deploy a Datasette branch preview to Vercel Closes #2070 --- .github/workflows/deploy-branch-preview.yml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..872aff71 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 2e43a14da195b3a4d4d413b217cdca0239844e26 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:35:34 -0700 Subject: [PATCH 6/9] datasette.utils.check_callable(obj) - refs #2078 --- datasette/utils/callable.py | 25 ++++++++++++++++++++ tests/test_utils_callable.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 datasette/utils/callable.py create mode 100644 tests/test_utils_callable.py diff --git a/datasette/utils/callable.py b/datasette/utils/callable.py new file mode 100644 index 00000000..5b8a30ac --- /dev/null +++ b/datasette/utils/callable.py @@ -0,0 +1,25 @@ +import asyncio +import types +from typing import NamedTuple, Any + + +class CallableStatus(NamedTuple): + is_callable: bool + is_async_callable: bool + + +def check_callable(obj: Any) -> CallableStatus: + if not callable(obj): + return CallableStatus(False, False) + + if isinstance(obj, type): + # It's a class + return CallableStatus(True, False) + + if isinstance(obj, types.FunctionType): + return CallableStatus(True, asyncio.iscoroutinefunction(obj)) + + if hasattr(obj, "__call__"): + return CallableStatus(True, asyncio.iscoroutinefunction(obj.__call__)) + + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/tests/test_utils_callable.py b/tests/test_utils_callable.py new file mode 100644 index 00000000..d1d0aac5 --- /dev/null +++ b/tests/test_utils_callable.py @@ -0,0 +1,46 @@ +from datasette.utils.callable import check_callable +import pytest + + +class AsyncClass: + async def __call__(self): + pass + + +class NotAsyncClass: + def __call__(self): + pass + + +class ClassNoCall: + pass + + +async def async_func(): + pass + + +def non_async_func(): + pass + + +@pytest.mark.parametrize( + "obj,expected_is_callable,expected_is_async_callable", + ( + (async_func, True, True), + (non_async_func, True, False), + (AsyncClass(), True, True), + (NotAsyncClass(), True, False), + (ClassNoCall(), False, False), + (AsyncClass, True, False), + (NotAsyncClass, True, False), + (ClassNoCall, True, False), + ("", False, False), + (1, False, False), + (str, True, False), + ), +) +def test_check_callable(obj, expected_is_callable, expected_is_async_callable): + status = check_callable(obj) + assert status.is_callable == expected_is_callable + assert status.is_async_callable == expected_is_async_callable From 9584879534ff0556e04e4c420262972884cac87b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:49:40 -0700 Subject: [PATCH 7/9] Rename callable.py to check_callable.py, refs #2078 --- datasette/utils/{callable.py => check_callable.py} | 0 tests/{test_utils_callable.py => test_utils_check_callable.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename datasette/utils/{callable.py => check_callable.py} (100%) rename tests/{test_utils_callable.py => test_utils_check_callable.py} (94%) diff --git a/datasette/utils/callable.py b/datasette/utils/check_callable.py similarity index 100% rename from datasette/utils/callable.py rename to datasette/utils/check_callable.py diff --git a/tests/test_utils_callable.py b/tests/test_utils_check_callable.py similarity index 94% rename from tests/test_utils_callable.py rename to tests/test_utils_check_callable.py index d1d0aac5..4f72f9ff 100644 --- a/tests/test_utils_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,4 +1,4 @@ -from datasette.utils.callable import check_callable +from datasette.utils.check_callable import check_callable import pytest From b49fa446d683ddcaf6faf2944dacc0d866bf2d70 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 15:05:58 -0700 Subject: [PATCH 8/9] --cors Access-Control-Max-Age: 3600, closes #2079 --- datasette/utils/__init__.py | 1 + docs/json_api.rst | 18 +++++++++++++++++- tests/test_api.py | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 925c6560..c388673d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1141,6 +1141,7 @@ def add_cors_headers(headers): headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" headers["Access-Control-Expose-Headers"] = "Link" headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS" + headers["Access-Control-Max-Age"] = "3600" _TILDE_ENCODING_SAFE = frozenset( diff --git a/docs/json_api.rst b/docs/json_api.rst index 7b130c58..c273c2a8 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,12 +454,28 @@ Enabling CORS ------------- If you start Datasette with the ``--cors`` option, each JSON endpoint will be -served with the following additional HTTP headers:: +served with the following additional HTTP headers: + +.. [[[cog + from datasette.utils import add_cors_headers + import textwrap + headers = {} + add_cors_headers(headers) + output = "\n".join("{}: {}".format(k, v) for k, v in headers.items()) + cog.out("\n::\n\n") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] + +:: Access-Control-Allow-Origin: * Access-Control-Allow-Headers: Authorization, Content-Type Access-Control-Expose-Headers: Link Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS + Access-Control-Max-Age: 3600 + +.. [[[end]]] This allows JavaScript running on any domain to make cross-origin requests to interact with the Datasette API. diff --git a/tests/test_api.py b/tests/test_api.py index 780e9fa5..247fdd5c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -941,6 +941,7 @@ def test_cors( assert ( response.headers["Access-Control-Allow-Methods"] == "GET, POST, HEAD, OPTIONS" ) + assert response.headers["Access-Control-Max-Age"] == "3600" # Same request to app_client_two_attached_databases_one_immutable # should not have those headers - I'm using that fixture because # regular app_client doesn't have immutable fixtures.db which means @@ -951,6 +952,7 @@ def test_cors( assert "Access-Control-Allow-Headers" not in response.headers assert "Access-Control-Expose-Headers" not in response.headers assert "Access-Control-Allow-Methods" not in response.headers + assert "Access-Control-Max-Age" not in response.headers @pytest.mark.parametrize( From dda99fc09fb0b5523948f6d481c6c051c1c7b5de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 17:18:43 -0700 Subject: [PATCH 9/9] New View base class (#2080) * New View base class, closes #2078 * Use new View subclass for PatternPortfolioView --- datasette/app.py | 40 +++++++++++++++++- datasette/views/base.py | 37 +++++++++++++++++ datasette/views/special.py | 19 +++++---- tests/test_base_view.py | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 tests/test_base_view.py diff --git a/datasette/app.py b/datasette/app.py index d7dace67..1f80c5a9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -17,6 +17,7 @@ import secrets import sys import threading import time +import types import urllib.parse from concurrent import futures from pathlib import Path @@ -1361,7 +1362,7 @@ class Datasette: r"/-/allow-debug$", ) add_route( - PatternPortfolioView.as_view(self), + wrap_view(PatternPortfolioView, self), r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") @@ -1673,7 +1674,42 @@ def _cleaner_task_str(task): return _cleaner_task_str_re.sub("", s) -def wrap_view(view_fn, datasette): +def wrap_view(view_fn_or_class, datasette): + is_function = isinstance(view_fn_or_class, types.FunctionType) + if is_function: + return wrap_view_function(view_fn_or_class, datasette) + else: + if not isinstance(view_fn_or_class, type): + raise ValueError("view_fn_or_class must be a function or a class") + return wrap_view_class(view_fn_or_class, datasette) + + +def wrap_view_class(view_class, datasette): + async def async_view_for_class(request, send): + instance = view_class() + if inspect.iscoroutinefunction(instance.__call__): + return await async_call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + else: + return call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + + return async_view_for_class + + +def wrap_view_function(view_fn, datasette): @functools.wraps(view_fn) async def async_view_fn(request, send): if inspect.iscoroutinefunction(view_fn): diff --git a/datasette/views/base.py b/datasette/views/base.py index 927d1aff..94645cd8 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -53,6 +53,43 @@ class DatasetteError(Exception): self.message_is_html = message_is_html +class View: + async def head(self, request, datasette): + if not hasattr(self, "get"): + return await self.method_not_allowed(request) + response = await self.get(request, datasette) + response.body = "" + return response + + async def method_not_allowed(self, request): + if ( + request.path.endswith(".json") + or request.headers.get("content-type") == "application/json" + ): + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) + else: + response = Response.text("Method not allowed", status=405) + return response + + async def options(self, request, datasette): + response = Response.text("ok") + response.headers["allow"] = ", ".join( + method.upper() + for method in ("head", "get", "post", "put", "patch", "delete") + if hasattr(self, method) + ) + return response + + async def __call__(self, request, datasette): + try: + handler = getattr(self, request.method.lower()) + except AttributeError: + return await self.method_not_allowed(request) + return await handler(request, datasette) + + class BaseView: ds = None has_json_alternate = True diff --git a/datasette/views/special.py b/datasette/views/special.py index 1aeb4be6..03e085d6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,7 +6,7 @@ from datasette.utils import ( tilde_encode, tilde_decode, ) -from .base import BaseView +from .base import BaseView, View import secrets import urllib @@ -57,13 +57,16 @@ class JsonDataView(BaseView): ) -class PatternPortfolioView(BaseView): - name = "patterns" - has_json_alternate = False - - async def get(self, request): - await self.ds.ensure_permissions(request.actor, ["view-instance"]) - return await self.render(["patterns.html"], request=request) +class PatternPortfolioView(View): + async def get(self, request, datasette): + await datasette.ensure_permissions(request.actor, ["view-instance"]) + return Response.html( + await datasette.render_template( + "patterns.html", + request=request, + view_name="patterns", + ) + ) class AuthTokenView(BaseView): diff --git a/tests/test_base_view.py b/tests/test_base_view.py new file mode 100644 index 00000000..2cd4d601 --- /dev/null +++ b/tests/test_base_view.py @@ -0,0 +1,84 @@ +from datasette.views.base import View +from datasette import Request, Response +from datasette.app import Datasette +import json +import pytest + + +class GetView(View): + async def get(self, request, datasette): + return Response.json( + { + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +class GetAndPostView(GetView): + async def post(self, request, datasette): + return Response.json( + { + "method": request.method, + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +@pytest.mark.asyncio +async def test_get_view(): + v = GetView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert post_response.body == "Method not allowed" + assert post_response.status == 405 + # And POST with .json extension + post_json_response = await v(Request.fake("/foo.json", method="POST"), datasette) + assert json.loads(post_json_response.body) == { + "ok": False, + "error": "Method not allowed", + } + assert post_json_response.status == 405 + + +@pytest.mark.asyncio +async def test_post_view(): + v = GetAndPostView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET, POST" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert json.loads(post_response.body) == { + "method": "POST", + "absolute_url": "http://localhost/", + "request_path": "/foo", + }