From f80ff9b07b5ecdfeca4aa81f5728812a22bfb019 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:16:48 -0700 Subject: [PATCH 01/81] min-height on .hd Now it should be the same size on the homepage as it is on pages with breadcrumbs --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 76ecdd8d..ece60d9e 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -90,6 +90,7 @@ table a:visited { background-color: #eee; overflow: hidden; box-sizing: border-box; + min-height: 2rem; } .hd p { margin: 0; From 787dd427de97dcbd0843611f1aef6d157d8bb0b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:21:50 -0700 Subject: [PATCH 02/81] white-space: pre-wrap for table SQL, closes #505 --- datasette/static/app.css | 4 ++++ datasette/templates/table.html | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index ece60d9e..80eda0e3 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -313,3 +313,7 @@ a.not-underlined { font-style: normal; font-size: 0.8em; } + +pre.wrapped-sql { + white-space: pre-wrap; +} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index c7913f60..1841300b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -184,11 +184,11 @@ {% endif %} {% if table_definition %} - 
{{ table_definition }}
+ 
{{ table_definition }}
{% endif %} {% if view_definition %} - 
{{ view_definition }}
+ 
{{ view_definition }}
{% endif %} {% endblock %} From 912ce848b9fa8b1642c800b446f504518bc39f2a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:25:38 -0700 Subject: [PATCH 03/81] Fix nav display on 500 page, closes #545 --- datasette/templates/500.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/templates/500.html b/datasette/templates/500.html index 809b2a71..46573f30 100644 --- a/datasette/templates/500.html +++ b/datasette/templates/500.html @@ -2,8 +2,14 @@ {% block title %}{% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}{% endblock %} +{% block nav %} +

+ home +

+ {{ super() }} +{% endblock %} + {% block content %} -
home

{% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}

From 9998f92cc05e6061a81af6cf194c3caa4d0759c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 16:19:02 -0700 Subject: [PATCH 04/81] Updated custom facet docs, closes #482 --- docs/plugins.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 3b3653cc..faa27daf 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -712,9 +712,9 @@ Each Facet subclass implements a new type of facet operation. The class should l # This key must be unique across all facet classes: type = "special" - async def suggest(self, sql, params, filtered_table_rows_count): + async def suggest(self): + # Use self.sql and self.params to suggest some facets suggested_facets = [] - # Perform calculations to suggest facets suggested_facets.append({ "name": column, # Or other unique name # Construct the URL that will enable this facet: @@ -726,8 +726,9 @@ Each Facet subclass implements a new type of facet operation. The class should l }) return suggested_facets - async def facet_results(self, sql, params): - # This should execute the facet operation and return results + async def facet_results(self): + # This should execute the facet operation and return results, again + # using self.sql and self.params as the starting point facet_results = {} facets_timed_out = [] # Do some calculations here... @@ -752,7 +753,7 @@ Each Facet subclass implements a new type of facet operation. The class should l return facet_results, facets_timed_out -See ``datasette/facets.py`` for examples of how these classes can work. +See `datasette/facets.py `__ for examples of how these classes can work. The plugin hook can then be used to register the new facet class like this: From c5542abba564a0b320a1201a8cc85b48c743005d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 16:21:11 -0700 Subject: [PATCH 05/81] Removed ManyToManyFacet for the moment, closes #550 --- datasette/facets.py | 189 +------------------------------------------ tests/test_facets.py | 60 +------------- 2 files changed, 2 insertions(+), 247 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 76d73e51..365d9c65 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -60,7 +60,7 @@ def load_facet_configs(request, table_metadata): @hookimpl def register_facet_classes(): - classes = [ColumnFacet, DateFacet, ManyToManyFacet] + classes = [ColumnFacet, DateFacet] if detect_json1(): classes.append(ArrayFacet) return classes @@ -476,190 +476,3 @@ class DateFacet(Facet): facets_timed_out.append(column) return facet_results, facets_timed_out - - -class ManyToManyFacet(Facet): - type = "m2m" - - async def suggest(self): - # This is calculated based on foreign key relationships to this table - # Are there any many-to-many tables pointing here? - suggested_facets = [] - db = self.ds.databases[self.database] - all_foreign_keys = await db.get_all_foreign_keys() - if not all_foreign_keys.get(self.table): - # It's probably a view - return [] - args = set(self.get_querystring_pairs()) - incoming = all_foreign_keys[self.table]["incoming"] - # Do any of these incoming tables have exactly two outgoing keys? - for fk in incoming: - other_table = fk["other_table"] - other_table_outgoing_foreign_keys = all_foreign_keys[other_table][ - "outgoing" - ] - if len(other_table_outgoing_foreign_keys) == 2: - destination_table = [ - t - for t in other_table_outgoing_foreign_keys - if t["other_table"] != self.table - ][0]["other_table"] - # Only suggest if it's not selected already - if ("_facet_m2m", destination_table) in args: - continue - suggested_facets.append( - { - "name": destination_table, - "type": "m2m", - "toggle_url": self.ds.absolute_url( - self.request, - path_with_added_args( - self.request, {"_facet_m2m": destination_table} - ), - ), - } - ) - return suggested_facets - - async def facet_results(self): - facet_results = {} - facets_timed_out = [] - args = set(self.get_querystring_pairs()) - facet_size = self.ds.config("default_facet_size") - db = self.ds.databases[self.database] - all_foreign_keys = await db.get_all_foreign_keys() - if not all_foreign_keys.get(self.table): - return [], [] - # We care about three tables: self.table, middle_table and destination_table - incoming = all_foreign_keys[self.table]["incoming"] - for source_and_config in self.get_configs(): - config = source_and_config["config"] - source = source_and_config["source"] - # The destination_table is specified in the _facet_m2m=xxx parameter - destination_table = config.get("column") or config["simple"] - # Find middle table - it has fks to self.table AND destination_table - fks = None - middle_table = None - for fk in incoming: - other_table = fk["other_table"] - other_table_outgoing_foreign_keys = all_foreign_keys[other_table][ - "outgoing" - ] - if ( - any( - o - for o in other_table_outgoing_foreign_keys - if o["other_table"] == destination_table - ) - and len(other_table_outgoing_foreign_keys) == 2 - ): - fks = other_table_outgoing_foreign_keys - middle_table = other_table - break - if middle_table is None or fks is None: - return [], [] - # Now that we have determined the middle_table, we need to figure out the three - # columns on that table which are relevant to us. These are: - # column_to_table - the middle_table column with a foreign key to self.table - # table_pk - the primary key column on self.table that is referenced - # column_to_destination - the column with a foreign key to destination_table - # - # It turns out we don't actually need the fourth obvious column: - # destination_pk = the primary key column on destination_table which is referenced - # - # These are both in the fks array - which now contains 2 foreign key relationships, e.g: - # [ - # {'other_table': 'characteristic', 'column': 'characteristic_id', 'other_column': 'pk'}, - # {'other_table': 'attractions', 'column': 'attraction_id', 'other_column': 'pk'} - # ] - column_to_table = None - table_pk = None - column_to_destination = None - for fk in fks: - if fk["other_table"] == self.table: - table_pk = fk["other_column"] - column_to_table = fk["column"] - elif fk["other_table"] == destination_table: - column_to_destination = fk["column"] - assert all((column_to_table, table_pk, column_to_destination)) - facet_sql = """ - select - {middle_table}.{column_to_destination} as value, - count(distinct {middle_table}.{column_to_table}) as count - from {middle_table} - where {middle_table}.{column_to_table} in ( - select {table_pk} from ({sql}) - ) - group by {middle_table}.{column_to_destination} - order by count desc limit {limit} - """.format( - sql=self.sql, - limit=facet_size + 1, - middle_table=escape_sqlite(middle_table), - column_to_destination=escape_sqlite(column_to_destination), - column_to_table=escape_sqlite(column_to_table), - table_pk=escape_sqlite(table_pk), - ) - try: - facet_rows_results = await self.ds.execute( - self.database, - facet_sql, - self.params, - truncate=False, - custom_time_limit=self.ds.config("facet_time_limit_ms"), - ) - facet_results_values = [] - facet_results[destination_table] = { - "name": destination_table, - "type": self.type, - "results": facet_results_values, - "hideable": source != "metadata", - "toggle_url": path_with_removed_args( - self.request, {"_facet_m2m": destination_table} - ), - "truncated": len(facet_rows_results) > facet_size, - } - facet_rows = facet_rows_results.rows[:facet_size] - - # Attempt to expand foreign keys into labels - values = [row["value"] for row in facet_rows] - expanded = await self.ds.expand_foreign_keys( - self.database, middle_table, column_to_destination, values - ) - - for row in facet_rows: - through = json.dumps( - { - "table": middle_table, - "column": column_to_destination, - "value": str(row["value"]), - }, - separators=(",", ":"), - sort_keys=True, - ) - selected = ("_through", through) in args - if selected: - toggle_path = path_with_removed_args( - self.request, {"_through": through} - ) - else: - toggle_path = path_with_added_args( - self.request, {"_through": through} - ) - facet_results_values.append( - { - "value": row["value"], - "label": expanded.get( - (column_to_destination, row["value"]), row["value"] - ), - "count": row["count"], - "toggle_url": self.ds.absolute_url( - self.request, toggle_path - ), - "selected": selected, - } - ) - except QueryInterrupted: - facets_timed_out.append(destination_table) - - return facet_results, facets_timed_out diff --git a/tests/test_facets.py b/tests/test_facets.py index b1037396..9169f666 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,8 +1,7 @@ -from datasette.facets import ColumnFacet, ArrayFacet, DateFacet, ManyToManyFacet +from datasette.facets import ColumnFacet, ArrayFacet, DateFacet from datasette.utils import detect_json1 from .fixtures import app_client # noqa from .utils import MockRequest -from collections import namedtuple import pytest @@ -303,60 +302,3 @@ async def test_date_facet_results(app_client): "truncated": False, } } == buckets - - -@pytest.mark.asyncio -async def test_m2m_facet_suggest(app_client): - facet = ManyToManyFacet( - app_client.ds, - MockRequest("http://localhost/"), - database="fixtures", - sql="select * from roadside_attractions", - table="roadside_attractions", - ) - suggestions = await facet.suggest() - assert [ - { - "name": "attraction_characteristic", - "type": "m2m", - "toggle_url": "http://localhost/?_facet_m2m=attraction_characteristic", - } - ] == suggestions - - -@pytest.mark.asyncio -async def test_m2m_facet_results(app_client): - facet = ManyToManyFacet( - app_client.ds, - MockRequest("http://localhost/?_facet_m2m=attraction_characteristic"), - database="fixtures", - sql="select * from roadside_attractions", - table="roadside_attractions", - ) - buckets, timed_out = await facet.facet_results() - assert [] == timed_out - assert { - "attraction_characteristic": { - "name": "attraction_characteristic", - "type": "m2m", - "results": [ - { - "value": 2, - "label": "Paranormal", - "count": 3, - "toggle_url": "http://localhost/?_facet_m2m=attraction_characteristic&_through=%7B%22column%22%3A%22characteristic_id%22%2C%22table%22%3A%22roadside_attraction_characteristics%22%2C%22value%22%3A%222%22%7D", - "selected": False, - }, - { - "value": 1, - "label": "Museum", - "count": 2, - "toggle_url": "http://localhost/?_facet_m2m=attraction_characteristic&_through=%7B%22column%22%3A%22characteristic_id%22%2C%22table%22%3A%22roadside_attraction_characteristics%22%2C%22value%22%3A%221%22%7D", - "selected": False, - }, - ], - "hideable": True, - "toggle_url": "/", - "truncated": False, - } - } == buckets From aa4cc99c0221a98850f8c801c329aac40f243b7b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 18:22:05 -0700 Subject: [PATCH 06/81] Removed facet-by-m2m from docs, refs #550 Will bring this back in #551 --- docs/facets.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/facets.rst b/docs/facets.rst index ddf69cb4..13b18bd0 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -129,17 +129,6 @@ The performance of facets can be greatly improved by adding indexes on the colum Enter ".help" for usage hints. sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state"); -.. _facet_by_m2m: - -Facet by many-to-many ---------------------- - -Datasette can detect many-to-many SQL tables - defined as SQL tables which have foreign key relationships to two other tables. - -If a many-to-many table exists pointing at the table you are currently viewing, Datasette will suggest you facet the table based on that relationship. - -Example here: `latest.datasette.io/fixtures/roadside_attractions?_facet_m2m=attraction_characteristic `__ - .. _facet_by_json_array: Facet by JSON array From 2d04986c4438cdfd3bb9d156d9dfcf830cb87b49 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 19:02:27 -0700 Subject: [PATCH 07/81] Added datasette-auth-github and datasette-cors plugins to Ecosystem Closes #548 --- docs/ecosystem.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 7d87fa85..cb6a3768 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -70,6 +70,11 @@ datasette-vega `datasette-vega `__ exposes the powerful `Vega `__ charting library, allowing you to construct line, bar and scatter charts against your data and share links to your visualizations. +datasette-auth-github +--------------------- + +`datasette-auth-github `__ adds an authentication layer to Datasette. Users will have to sign in using their GitHub account before they can view data or interact with Datasette. You can also use it to restrict access to specific GitHub users, or to members of specified GitHub `organizations `__ or `teams `__. + datasette-json-html ------------------- @@ -114,3 +119,8 @@ datasette-bplist ---------------- `datasette-bplist `__ provides tools for working with Apple's binary plist format embedded in SQLite database tables. If you use OS X you already have dozens of SQLite databases hidden away in your ``~/Library`` folder that include data in this format - this plugin allows you to view the decoded data and run SQL queries against embedded values using a ``bplist_to_json(value)`` custom SQL function. + +datasette-cors +-------------- + +`datasette-cors `__ allows you to configure `CORS headers `__ for your Datasette instance. You can use this to enable JavaScript running on a whitelisted set of domains to make ``fetch()`` calls to the JSON API provided by your Datasette instance. \ No newline at end of file From 973f8f139df6ad425354711052cfc2256de2e522 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 19:06:31 -0700 Subject: [PATCH 08/81] --plugin-secret option for datasette publish Closes #543 Also added new --show-files option to publish now and publish cloudrun - handy for debugging. --- datasette/publish/cloudrun.py | 51 +++++++++++++++++---- datasette/publish/common.py | 15 +++++++ datasette/publish/heroku.py | 40 +++++++++++++---- datasette/publish/now.py | 50 +++++++++++++++++---- datasette/utils/__init__.py | 10 +++++ docs/datasette-publish-cloudrun-help.txt | 42 ++++++++++-------- docs/datasette-publish-heroku-help.txt | 37 +++++++++------- docs/datasette-publish-nowv1-help.txt | 46 ++++++++++--------- docs/plugins.rst | 11 ++++- docs/publish.rst | 9 ++++ tests/test_publish_cloudrun.py | 54 +++++++++++++++++++++++ tests/test_publish_heroku.py | 46 ++++++++++++++++++- tests/test_publish_now.py | 56 ++++++++++++++++++++++++ 13 files changed, 381 insertions(+), 86 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 436b5d2b..32c9cd2a 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,6 +1,7 @@ from datasette import hookimpl import click import json +import os from subprocess import check_call, check_output from .common import ( @@ -24,6 +25,11 @@ def publish_subcommand(publish): "--service", default="", help="Cloud Run service to deploy (or over-write)" ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + @click.option( + "--show-files", + is_flag=True, + help="Output the generated Dockerfile and metadata.json", + ) def cloudrun( files, metadata, @@ -33,6 +39,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, @@ -44,6 +51,7 @@ def publish_subcommand(publish): name, service, spatialite, + show_files, ): fail_if_publish_binary_not_installed( "gcloud", "Google Cloud", "https://cloud.google.com/sdk/" @@ -52,6 +60,30 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_docker_directory( files, name, @@ -64,16 +96,17 @@ def publish_subcommand(publish): install, spatialite, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, + environment_variables, ): + if show_files: + if os.path.exists("metadata.json"): + print("=== metadata.json ===\n") + print(open("metadata.json").read()) + print("\n==== Dockerfile ====\n") + print(open("Dockerfile").read()) + print("\n====================\n") + image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( diff --git a/datasette/publish/common.py b/datasette/publish/common.py index a31eef02..5bbbf613 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -41,6 +41,14 @@ def add_common_publish_arguments_and_options(subcommand): help="Additional packages (e.g. plugins) to install", multiple=True, ), + click.option( + "--plugin-secret", + nargs=3, + type=(str, str, str), + callback=validate_plugin_secret, + multiple=True, + help="Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx", + ), click.option( "--version-note", help="Additional note to show on /-/versions" ), @@ -76,3 +84,10 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): err=True, ) sys.exit(1) + + +def validate_plugin_secret(ctx, param, value): + for plugin_name, plugin_setting, setting_value in value: + if "'" in setting_value: + raise click.BadParameter("--plugin-secret cannot contain single quotes") + return value diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 5705500f..34d1f773 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -33,6 +33,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, @@ -61,6 +62,30 @@ def publish_subcommand(publish): ) call(["heroku", "plugins:install", "heroku-builds"]) + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_heroku_directory( files, name, @@ -72,15 +97,7 @@ def publish_subcommand(publish): static, install, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, ): app_name = None if name: @@ -104,6 +121,11 @@ def publish_subcommand(publish): create_output = check_output(cmd).decode("utf8") app_name = json.loads(create_output)["name"] + for key, value in environment_variables.items(): + call( + ["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)] + ) + call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"]) diff --git a/datasette/publish/now.py b/datasette/publish/now.py index 38add86e..d7831c80 100644 --- a/datasette/publish/now.py +++ b/datasette/publish/now.py @@ -1,6 +1,7 @@ from datasette import hookimpl import click import json +import os from subprocess import run, PIPE from .common import ( @@ -24,6 +25,11 @@ def publish_subcommand(publish): @click.option("--token", help="Auth token to use for deploy") @click.option("--alias", multiple=True, help="Desired alias e.g. yoursite.now.sh") @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + @click.option( + "--show-files", + is_flag=True, + help="Output the generated Dockerfile and metadata.json", + ) def nowv1( files, metadata, @@ -33,6 +39,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, @@ -46,6 +53,7 @@ def publish_subcommand(publish): token, alias, spatialite, + show_files, ): fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") if extra_options: @@ -54,6 +62,30 @@ def publish_subcommand(publish): extra_options = "" extra_options += "--config force_https_urls:on" + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_docker_directory( files, name, @@ -66,15 +98,8 @@ def publish_subcommand(publish): install, spatialite, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, + environment_variables, ): now_json = {"version": 1} open("now.json", "w").write(json.dumps(now_json, indent=4)) @@ -88,6 +113,13 @@ def publish_subcommand(publish): else: done = run("now", stdout=PIPE) deployment_url = done.stdout + if show_files: + if os.path.exists("metadata.json"): + print("=== metadata.json ===\n") + print(open("metadata.json").read()) + print("\n==== Dockerfile ====\n") + print(open("Dockerfile").read()) + print("\n====================\n") if alias: # I couldn't get --target=production working, so I call # 'now alias' with arguments directly instead - but that diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 17a4d595..d92d0cd5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -272,6 +272,7 @@ def make_dockerfile( install, spatialite, version_note, + environment_variables=None, ): cmd = ["datasette", "serve", "--host", "0.0.0.0"] for filename in files: @@ -307,11 +308,18 @@ FROM python:3.6 COPY . /app WORKDIR /app {spatialite_extras} +{environment_variables} RUN pip install -U {install_from} RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT 8001 EXPOSE 8001 CMD {cmd}""".format( + environment_variables="\n".join( + [ + "ENV {} '{}'".format(key, value) + for key, value in (environment_variables or {}).items() + ] + ), files=" ".join(files), cmd=cmd, install_from=" ".join(install), @@ -333,6 +341,7 @@ def temporary_docker_directory( spatialite, version_note, extra_metadata=None, + environment_variables=None, ): extra_metadata = extra_metadata or {} tmp = tempfile.TemporaryDirectory() @@ -361,6 +370,7 @@ def temporary_docker_directory( install, spatialite, version_note, + environment_variables, ) os.chdir(datasette_dir) if metadata_content: diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index fc7d44d5..6cdc87eb 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -3,22 +3,26 @@ $ datasette publish cloudrun --help Usage: datasette publish cloudrun [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when building - --service TEXT Cloud Run service to deploy (or over-write) - --spatialite Enable SpatialLite extension - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when building + --service TEXT Cloud Run service to deploy (or over-write) + --spatialite Enable SpatialLite extension + --show-files Output the generated Dockerfile and metadata.json + --help Show this message and exit. diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index cd9af09b..88d387a6 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -3,20 +3,23 @@ $ datasette publish heroku --help Usage: datasette publish heroku [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when deploying - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when deploying + --help Show this message and exit. diff --git a/docs/datasette-publish-nowv1-help.txt b/docs/datasette-publish-nowv1-help.txt index a5417d71..c2bf23f1 100644 --- a/docs/datasette-publish-nowv1-help.txt +++ b/docs/datasette-publish-nowv1-help.txt @@ -3,24 +3,28 @@ $ datasette publish nowv1 --help Usage: datasette publish nowv1 [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when deploying - --force Pass --force option to now - --token TEXT Auth token to use for deploy - --alias TEXT Desired alias e.g. yoursite.now.sh - --spatialite Enable SpatialLite extension - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when deploying + --force Pass --force option to now + --token TEXT Auth token to use for deploy + --alias TEXT Desired alias e.g. yoursite.now.sh + --spatialite Enable SpatialLite extension + --show-files Output the generated Dockerfile and metadata.json + --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index faa27daf..1d4f1e1a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -219,6 +219,8 @@ Here is an example of some plugin configuration for a specific table:: This tells the ``datasette-cluster-map`` column which latitude and longitude columns should be used for a table called ``Street_Tree_List`` inside a database file called ``sf-trees.db``. +.. _plugins_configuration_secret: + Secret configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -236,7 +238,6 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } - **As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this:: { @@ -249,6 +250,14 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } +If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: + + $ datasette publish heroku my_database.db \ + --name my-heroku-app-demo \ + --install=datasette-auth-github \ + --plugin-secret datasette-auth-github client_id your_client_id \ + --plugin-secret datasette-auth-github client_secret your_client_secret + Writing plugins that accept configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/publish.rst b/docs/publish.rst index c9039734..009ae199 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -6,6 +6,8 @@ Datasette includes tools for publishing and deploying your data to the internet. The ``datasette publish`` command will deploy a new Datasette instance containing your databases directly to a Heroku, Google Cloud or Zeit Now hosting account. You can also use ``datasette package`` to create a Docker image that bundles your databases together with the datasette application that is used to serve them. +.. _cli_publish: + datasette publish ================= @@ -99,6 +101,13 @@ You can also specify plugins you would like to install. For example, if you want datasette publish nowv1 mydatabase.db --install=datasette-vega +If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: + + $ datasette publish heroku my_database.db \ + --name my-heroku-app-demo \ + --install=datasette-auth-github \ + --plugin-secret datasette-auth-github client_id your_client_id \ + --plugin-secret datasette-auth-github client_secret your_client_secret datasette package ================= diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index d26786ce..1e9bb830 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import json @mock.patch("shutil.which") @@ -46,3 +47,56 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): ), ] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.cloudrun.check_output") +@mock.patch("datasette.publish.cloudrun.check_call") +def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): + mock_which.return_value = True + mock_output.return_value = "myproject" + + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + "--show-files", + ], + ) + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = """FROM python:3.6 +COPY . /app +WORKDIR /app + +ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' +RUN pip install -U datasette +RUN datasette inspect test.db --inspect-file inspect-data.json +ENV PORT 8001 +EXPOSE 8001 +CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --port $PORT""".strip() + assert expected == dockerfile + metadata = ( + result.output.split("=== metadata.json ===\n")[1] + .split("\n==== Dockerfile ====\n")[0] + .strip() + ) + assert { + "plugins": { + "datasette-auth-github": { + "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"} + } + } + } == json.loads(metadata) diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 08fdeaea..4cd66219 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -46,7 +46,7 @@ def test_publish_heroku_invalid_database(mock_which): @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") def test_publish_heroku(mock_call, mock_check_output, mock_which): - mock_which.return_varue = True + mock_which.return_value = True mock_check_output.side_effect = lambda s: { "['heroku', 'plugins']": b"heroku-builds", "['heroku', 'apps:list', '--json']": b"[]", @@ -60,3 +60,47 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): mock_call.assert_called_once_with( ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.heroku.check_output") +@mock.patch("datasette.publish.heroku.call") +def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which): + mock_which.return_value = True + mock_check_output.side_effect = lambda s: { + "['heroku', 'plugins']": b"heroku-builds", + "['heroku', 'apps:list', '--json']": b"[]", + "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', + }[repr(s)] + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "heroku", + "test.db", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + ], + ) + assert 0 == result.exit_code, result.output + mock_call.assert_has_calls( + [ + mock.call( + [ + "heroku", + "config:set", + "-a", + "f", + "DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id", + ] + ), + mock.call( + ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + ), + ] + ) diff --git a/tests/test_publish_now.py b/tests/test_publish_now.py index fa1ab30a..72aa71db 100644 --- a/tests/test_publish_now.py +++ b/tests/test_publish_now.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import json import subprocess @@ -105,3 +106,58 @@ def test_publish_now_multiple_aliases(mock_run, mock_which): ), ] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.now.run") +def test_publish_now_plugin_secrets(mock_run, mock_which): + mock_which.return_value = True + mock_run.return_value = mock.Mock(0) + mock_run.return_value.stdout = b"https://demo.example.com/" + + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "now", + "test.db", + "--token", + "XXX", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + "--show-files", + ], + ) + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = """FROM python:3.6 +COPY . /app +WORKDIR /app + +ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' +RUN pip install -U datasette +RUN datasette inspect test.db --inspect-file inspect-data.json +ENV PORT 8001 +EXPOSE 8001 +CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --config force_https_urls:on --port $PORT""".strip() + assert expected == dockerfile + metadata = ( + result.output.split("=== metadata.json ===\n")[1] + .split("\n==== Dockerfile ====\n")[0] + .strip() + ) + assert { + "plugins": { + "datasette-auth-github": { + "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"} + } + } + } == json.loads(metadata) From fb7ee8e0ad59a15083234a48e935525f6e7257dd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 20:14:27 -0700 Subject: [PATCH 09/81] Changelog for 0.29 release --- docs/changelog.rst | 94 +++++++++++++++++++++++++++++++++++++++ docs/custom_templates.rst | 2 + docs/performance.rst | 2 + docs/publish.rst | 2 + 4 files changed, 100 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4be3f2d..93a42718 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,100 @@ Changelog ========= +.. _v0_29: + +0.29 (2019-07-07) +----------------- + +ASGI, new plugin hooks, facet by date and much, much more... + +ASGI +~~~~ + +`ASGI `__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 `__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn `__ and no longer depends on `Sanic `__. + +I wrote about the significance of this change in `Porting Datasette to ASGI, and Turtles all the way down `__. + +The most exciting consequence of this change is that Datasette plugins can now take advantage of the ASGI standard. + +New plugin hook: asgi_wrapper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`plugin_asgi_wrapper` plugin hook allows plugins to entirely wrap the Datasette ASGI application in their own ASGI middleware. (`#520 `__) + +Two new plugins take advantage of this hook: + +* `datasette-auth-github `__ adds a authentication layer: users will have to sign in using their GitHub account before they can view data or interact with Datasette. You can also use it to restrict access to specific GitHub users, or to members of specified GitHub `organizations `__ or `teams `__. + +* `datasette-cors `__ allows you to configure `CORS headers `__ for your Datasette instance. You can use this to enable JavaScript running on a whitelisted set of domains to make ``fetch()`` calls to the JSON API provided by your Datasette instance. + +New plugin hook: extra_template_vars +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). + +Secret plugin configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Plugins like `datasette-auth-github `__ need a safe way to set secret configuration options. Since the default mechanism for configuring plugins exposes those settings in ``/-/metadata`` a new mechanism was needed. :ref:`plugins_configuration_secret` describes how plugins can now specify that their settings should be read from a file or an environment variable:: + + { + "plugins": { + "datasette-auth-github": { + "client_secret": { + "$env": "GITHUB_CLIENT_SECRET" + } + } + } + } + +These plugin secrets can be set directly using ``datasette publish``. See :ref:`publish_custom_metadata_and_plugins` for details. (`#538 `__ and `#543 `__) + +Facet by date +~~~~~~~~~~~~~ + +If a column contains datetime values, Datasette can now facet that column by date. (`#481 `__) + +.. _v0_29_medium_changes: + +Easier custom templates for table rows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want to customize the display of individual table rows, you can do so using a ``_table.html`` template include that looks something like this:: + + {% for row in display_rows %} +
+

{{ row["title"] }}

+

{{ row["description"] }} +

Category: {{ row.display("category_id") }}

+
+ {% endfor %} + +This is a **backwards incompatible change**. If you previously had a custom template called ``_rows_and_columns.html`` you need to rename it to ``_table.html``. + +See :ref:`customization_custom_templates` for full details. + +?_through= for joins through many-to-many tables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The new ``?_through={json}`` argument to the Table view allows records to be filtered based on a many-to-many relationship. See :ref:`json_api_table_arguments` for full documentation - here's `an example `__. (`#355 `__) + +This feature was added to help support `facet by many-to-many `__, which isn't quite ready yet but will be coming in the next Datasette release. + +Small changes +~~~~~~~~~~~~~ + +* Databases published using ``datasette publish`` now open in :ref:`performance_immutable_mode`. (`#469 `__) +* ``?col__date=`` now works for columns containing spaces +* Automatic label detection (for deciding which column to show when linking to a foreign key) has been improved. (`#485 `__) +* Fixed bug where pagination broke when combined with an expanded foreign key. (`#489 `__) +* Contributors can now run ``pip install -e .[docs]`` to get all of the dependencies needed to build the documentation, including ``cd docs && make livehtml`` support. +* Datasette's dependencies are now all specified using the ``~=`` match operator. (`#532 `__) +* ``white-space: pre-wrap`` now used for table creation SQL. (`#505 `__) + + +`Full list of commits `__ between 0.28 and 0.29. + .. _v0_28: 0.28 (2019-05-19) diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 47271542..5cabe152 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -102,6 +102,8 @@ database column they are representing, for example:: +.. _customization_custom_templates: + Custom templates ---------------- diff --git a/docs/performance.rst b/docs/performance.rst index 741c9a92..580d7684 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -7,6 +7,8 @@ Datasette runs on top of SQLite, and SQLite has excellent performance. For smal That said, there are a number of tricks you can use to improve Datasette's performance. +.. _performance_immutable_mode: + Immutable mode -------------- diff --git a/docs/publish.rst b/docs/publish.rst index 009ae199..304be8ef 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -81,6 +81,8 @@ You can also use custom domains, if you `first register them with Zeit Now Date: Sun, 7 Jul 2019 21:36:27 -0700 Subject: [PATCH 10/81] News: Datasette 0.29, datasette-auth-github, datasette-cors --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 91b42753..9998a972 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... + * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. + * [datasette-cors](https://github.com/simonw/datasette-cors) - a plugin that lets you configure CORS access from a list of domains (or a set of domain wildcards) so you can make JavaScript calls to a Datasette instance from a specific set of other hosts. * 23rd June 2019: [Porting Datasette to ASGI, and Turtles all the way down](https://simonwillison.net/2019/Jun/23/datasette-asgi/) * 21st May 2019: The anonymized raw data from [the Stack Overflow Developer Survey 2019](https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/) has been [published in partnership with Glitch](https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/), powered by Datasette. * 19th May 2019: [Datasette 0.28](https://datasette.readthedocs.io/en/stable/changelog.html#v0-28) - a salmagundi of new features! From 9ca860e54fe480d0a365c0c1d8d085926d12be1e Mon Sep 17 00:00:00 2001 From: Abdus Date: Thu, 11 Jul 2019 19:07:44 +0300 Subject: [PATCH 11/81] Add support for running datasette as a module (#556) python -m datasette Thanks, @abdusco --- datasette/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 datasette/__main__.py diff --git a/datasette/__main__.py b/datasette/__main__.py new file mode 100644 index 00000000..4adef844 --- /dev/null +++ b/datasette/__main__.py @@ -0,0 +1,4 @@ +from datasette.cli import cli + +if __name__ == "__main__": + cli() From 74ecf8a7cc45cabf369e510c7214f5ed85c8c6d8 Mon Sep 17 00:00:00 2001 From: Abdus Date: Thu, 11 Jul 2019 19:13:19 +0300 Subject: [PATCH 12/81] Fix static mounts using relative paths and prevent traversal exploits (#554) Thanks, @abdusco! Closes #555 --- datasette/utils/__init__.py | 3 ++- datasette/utils/asgi.py | 6 +++++- tests/test_html.py | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d92d0cd5..87147baf 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -735,7 +735,8 @@ class StaticMount(click.ParamType): param, ctx, ) - path, dirpath = value.split(":") + path, dirpath = value.split(":", 1) + dirpath = os.path.abspath(dirpath) if not os.path.exists(dirpath) or not os.path.isdir(dirpath): self.fail("%s is not a valid directory path" % value, param, ctx) return path, dirpath diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 38ffc072..db7f9004 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -300,7 +300,11 @@ async def asgi_send_file( def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): async def inner_static(scope, receive, send): path = scope["url_route"]["kwargs"]["path"] - full_path = (Path(root_path) / path).absolute() + try: + full_path = (Path(root_path) / path).resolve().absolute() + except FileNotFoundError: + await asgi_send_html(send, "404", 404) + return # Ensure full_path is within root_path to avoid weird "../" tricks try: full_path.relative_to(root_path) diff --git a/tests/test_html.py b/tests/test_html.py index f76f98b9..0a6df984 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -67,6 +67,8 @@ def test_static_mounts(): assert response.status == 200 response = client.get("/custom-static/not_exists.py") assert response.status == 404 + response = client.get("/custom-static/../LICENSE") + assert response.status == 404 def test_memory_database_page(): From cc27857c722c172b3c9bd93c92f02e19f2a55d6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jul 2019 09:14:24 -0700 Subject: [PATCH 13/81] Removed unused variable --- datasette/utils/asgi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index db7f9004..466c5b89 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -311,7 +311,6 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): except ValueError: await asgi_send_html(send, "404", 404) return - first = True try: await asgi_send_file(send, full_path, chunk_size=chunk_size) except FileNotFoundError: From 2a94f3719fb2c4335fcda374fa92f87272b02d34 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jul 2019 09:17:55 -0700 Subject: [PATCH 14/81] Release 0.29.1 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 93a42718..ad2a577a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_29_1: + +0.29.1 (2019-07-11) +------------------- + +- Fixed bug with static mounts using relative paths which could lead to traversal exploits (`#555 `__) - thanks Abdussamet Kocak! + .. _v0_29: 0.29 (2019-07-07) From f2006cca80040871439055ae6ccbc14e589bdf4b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jul 2019 09:27:28 -0700 Subject: [PATCH 15/81] Updated release notes --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ad2a577a..01f0230b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Changelog ------------------- - Fixed bug with static mounts using relative paths which could lead to traversal exploits (`#555 `__) - thanks Abdussamet Kocak! +- Datasette can now be run as a module: ``python -m datasette`` (`#556 `__) - thanks, Abdussamet Kocak! .. _v0_29: From d224ee2c98ac39c2c6e21a0ac0c62e5c3e1ccd11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 15:34:57 -0700 Subject: [PATCH 16/81] Bump to uvicorn 0.8.4 (#559) https://github.com/encode/uvicorn/commits/0.8.4 Query strings will now be included in log files: https://github.com/encode/uvicorn/pull/384 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 254859b0..cbe545a1 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( "hupper~=1.0", "pint~=0.8.1", "pluggy~=0.12.0", - "uvicorn~=0.8.1", + "uvicorn~=0.8.4", "aiofiles~=0.4.0", ], entry_points=""" From afc2e4260ab8b28e132c834185c5294fb27543f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 18:42:35 -0700 Subject: [PATCH 17/81] News: Single sign-on against GitHub using ASGI middleware --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9998a972..59a6649e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. * [datasette-cors](https://github.com/simonw/datasette-cors) - a plugin that lets you configure CORS access from a list of domains (or a set of domain wildcards) so you can make JavaScript calls to a Datasette instance from a specific set of other hosts. From 5ed450a3328bd6a6a918474eeb5446d8a704df1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 19:05:39 -0700 Subject: [PATCH 18/81] Fixed breadcrumbs on custom query page --- datasette/templates/query.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b4b759a5..7c6c59f3 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -18,9 +18,15 @@ {% block body_class %}query db-{{ database|to_css_class }}{% endblock %} -{% block content %} -
home / {{ database }}
+{% block nav %} +

+ home / + {{ database }} +

+ {{ super() }} +{% endblock %} +{% block content %}

{{ metadata.title or database }}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} From 90d4f497f9b3f6a5882937c91fddb496ac3e7368 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 19:49:24 -0700 Subject: [PATCH 19/81] Fix plus test for unicode characters in custom query name, closes #558 --- datasette/utils/asgi.py | 7 ++++--- tests/fixtures.py | 29 ++++++++++++++++------------- tests/test_api.py | 5 +++++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 466c5b89..eaf3428d 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -45,9 +45,10 @@ class Request: @property def path(self): - return ( - self.scope.get("raw_path", self.scope["path"].encode("latin-1")) - ).decode("latin-1") + if "raw_path" in self.scope: + return self.scope["raw_path"].decode("latin-1") + else: + return self.scope["path"].decode("utf-8") @property def query_string(self): diff --git a/tests/fixtures.py b/tests/fixtures.py index d686142b..dac28dc0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -12,7 +12,7 @@ import sys import string import tempfile import time -from urllib.parse import unquote +from urllib.parse import unquote, quote # This temp file is used by one of the plugin config tests @@ -49,18 +49,20 @@ class TestClient: if "?" in path: path, _, query_string = path.partition("?") query_string = query_string.encode("utf8") - instance = ApplicationCommunicator( - self.asgi_app, - { - "type": "http", - "http_version": "1.0", - "method": method, - "path": unquote(path), - "raw_path": path.encode("ascii"), - "query_string": query_string, - "headers": [[b"host", b"localhost"]], - }, - ) + if "%" in path: + raw_path = path.encode("latin-1") + else: + raw_path = quote(path, safe="/:,").encode("latin-1") + scope = { + "type": "http", + "http_version": "1.0", + "method": method, + "path": unquote(path), + "raw_path": raw_path, + "query_string": query_string, + "headers": [[b"host", b"localhost"]], + } + instance = ApplicationCommunicator(self.asgi_app, scope) await instance.send_input({"type": "http.request"}) # First message back should be response.start with headers and status messages = [] @@ -291,6 +293,7 @@ METADATA = { }, }, "queries": { + "𝐜𝐢𝐭𝐢𝐞𝐬": "select id, name from facet_cities order by id limit 1;", "pragma_cache_size": "PRAGMA cache_size;", "neighborhood_search": { "sql": """ diff --git a/tests/test_api.py b/tests/test_api.py index a32ed5e3..cc00b780 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1613,6 +1613,11 @@ def test_infinity_returned_as_invalid_json_if_requested(app_client): ] == response.json +def test_custom_query_with_unicode_characters(app_client): + response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array") + assert [{"id": 1, "name": "San Francisco"}] == response.json + + def test_trace(app_client): response = app_client.get("/fixtures/simple_primary_key.json?_trace=1") data = response.json From 6abe6faff6b035e9334dd05f8c741ae9b7a47440 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 20:04:05 -0700 Subject: [PATCH 20/81] Release 0.9.2 --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01f0230b..d04ae2ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_29_2: + +0.29.2 (2019-07-13) +------------------- + +- Bumped `Uvicorn `__ to 0.8.4, fixing a bug where the querystring was not included in the server logs. (`#559 `__) +- Fixed bug where the navigation breadcrumbs were not displayed correctly on the page for a custom query. (`#558 `__) +- Fixed bug where custom query names containing unicode characters caused errors. + .. _v0_29_1: 0.29.1 (2019-07-11) From a9453c4dda70bbf5122835e68f63db6ecbe1a6fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jul 2019 20:38:40 -0700 Subject: [PATCH 21/81] Fixed CodeMirror on database page, closes #560 --- datasette/templates/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index f168db97..a934f336 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -25,7 +25,7 @@ {% if config.allow_sql %}

Custom SQL query

-

+

{% endif %} From 27cb29365c9f5f6f1492968d1268497193ed75a2 Mon Sep 17 00:00:00 2001 From: Min ho Kim Date: Fri, 26 Jul 2019 20:25:44 +1000 Subject: [PATCH 22/81] Fix numerous typos (#561) Thanks, @minho42! --- datasette/_version.py | 2 +- datasette/views/base.py | 2 +- docs/changelog.rst | 6 +++--- docs/metadata.rst | 2 +- docs/performance.rst | 2 +- versioneer.py | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/datasette/_version.py b/datasette/_version.py index a12f24aa..5783f30f 100644 --- a/datasette/_version.py +++ b/datasette/_version.py @@ -409,7 +409,7 @@ def render_pep440_old(pieces): The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: diff --git a/datasette/views/base.py b/datasette/views/base.py index 2c14be57..db1d69d9 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -387,7 +387,7 @@ class DataView(BaseView): return await self.as_csv(request, database, hash, **kwargs) if _format is None: - # HTML views default to expanding all foriegn key labels + # HTML views default to expanding all foreign key labels kwargs["default_labels"] = True extra_template_data = {} diff --git a/docs/changelog.rst b/docs/changelog.rst index d04ae2ca..69f799de 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -142,7 +142,7 @@ Datasette can still run against immutable files and gains numerous performance b Faceting improvements, and faceting plugins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Datasette :ref:`facets` provide an intuitive way to quickly summarize and interact with data. Previously the only supported faceting technique was column faceting, but 0.28 introduces two powerful new capibilities: facet-by-JSON-array and the ability to define further facet types using plugins. +Datasette :ref:`facets` provide an intuitive way to quickly summarize and interact with data. Previously the only supported faceting technique was column faceting, but 0.28 introduces two powerful new capabilities: facet-by-JSON-array and the ability to define further facet types using plugins. Facet by array (`#359 `__) is only available if your SQLite installation provides the ``json1`` extension. Datasette will automatically detect columns that contain JSON arrays of values and offer a faceting interface against those columns - useful for modelling things like tags without needing to break them out into a new table. See :ref:`facet_by_json_array` for more. @@ -153,7 +153,7 @@ The new :ref:`plugin_register_facet_classes` plugin hook (`#445 `__ is a brand new serverless hosting platform from Google, which allows you to build a Docker container which will run only when HTTP traffic is recieved and will shut down (and hence cost you nothing) the rest of the time. It's similar to Zeit's Now v1 Docker hosting platform which sadly is `no longer accepting signups `__ from new users. +`Google Cloud Run `__ is a brand new serverless hosting platform from Google, which allows you to build a Docker container which will run only when HTTP traffic is received and will shut down (and hence cost you nothing) the rest of the time. It's similar to Zeit's Now v1 Docker hosting platform which sadly is `no longer accepting signups `__ from new users. The new ``datasette publish cloudrun`` command was contributed by Romain Primet (`#434 `__) and publishes selected databases to a new Datasette instance running on Google Cloud Run. @@ -592,7 +592,7 @@ Mostly new work on the :ref:`plugins` mechanism: plugins can now bundle static a - Longer time limit for test_paginate_compound_keys It was failing intermittently in Travis - see `#209 `_ -- Use application/octet-stream for downloadable databses +- Use application/octet-stream for downloadable databases - Updated PyPI classifiers - Updated PyPI link to pypi.org diff --git a/docs/metadata.rst b/docs/metadata.rst index 0a2aa219..5d9155ea 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -62,7 +62,7 @@ Each of the top-level metadata fields can be used at the database and table leve Source, license and about ------------------------- -The three visible metadata fields you can apply to everything, specific databases or specific tables are source, license and about. All three are optionaly. +The three visible metadata fields you can apply to everything, specific databases or specific tables are source, license and about. All three are optional. **source** and **source_url** should be used to indicate where the underlying data came from. diff --git a/docs/performance.rst b/docs/performance.rst index 580d7684..d7f852d5 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -39,7 +39,7 @@ Then later you can start Datasette against the ``counts.json`` file and use it t datasette -i data.db --inspect-file=counts.json -You need to use the ``-i`` immutable mode agaist the databse file here or the counts from the JSON file will be ignored. +You need to use the ``-i`` immutable mode against the databse file here or the counts from the JSON file will be ignored. You will rarely need to use this optimization in every-day use, but several of the ``datasette publish`` commands described in :ref:`publishing` use this optimization for better performance when deploying a database file to a hosting provider. diff --git a/versioneer.py b/versioneer.py index 64fea1c8..e2e8a55b 100644 --- a/versioneer.py +++ b/versioneer.py @@ -180,7 +180,7 @@ two common reasons why `setup.py` might not be in the root: `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. + provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs @@ -805,7 +805,7 @@ def render_pep440_old(pieces): The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -1306,7 +1306,7 @@ def render_pep440_old(pieces): The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: From f04deebec4f3842f7bd610cd5859de529f77d50e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Jul 2019 16:07:44 +0300 Subject: [PATCH 23/81] Refactored connection logic to database.connect() --- datasette/app.py | 15 +-------------- datasette/database.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1a41c1c6..501d1467 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -470,20 +470,7 @@ class Datasette: def in_thread(): conn = getattr(connections, db_name, None) if not conn: - db = self.databases[db_name] - if db.is_memory: - conn = sqlite3.connect(":memory:") - else: - # mode=ro or immutable=1? - if db.is_mutable: - qs = "mode=ro" - else: - qs = "immutable=1" - conn = sqlite3.connect( - "file:{}?{}".format(db.path, qs), - uri=True, - check_same_thread=False, - ) + conn = self.databases[db_name].connect() self.prepare_connection(conn) setattr(connections, db_name, conn) return fn(conn) diff --git a/datasette/database.py b/datasette/database.py index e4915770..7e6f7245 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -33,6 +33,18 @@ class Database: for key, value in self.ds.inspect_data[self.name]["tables"].items() } + def connect(self): + if self.is_memory: + return sqlite3.connect(":memory:") + # mode=ro or immutable=1? + if self.is_mutable: + qs = "mode=ro" + else: + qs = "immutable=1" + return sqlite3.connect( + "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False + ) + @property def size(self): if self.is_memory: From 2dc5c8dc259a0606162673d394ba8cc1c6f54428 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Sep 2019 17:32:27 -0700 Subject: [PATCH 24/81] detect_fts now works with alternative table escaping (#571) Fixes #570. See also https://github.com/simonw/sqlite-utils/pull/57 --- datasette/utils/__init__.py | 1 + tests/test_utils.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 87147baf..115c4cbe 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -469,6 +469,7 @@ def detect_fts_sql(table): where rootpage = 0 and ( sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%' + or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%' or ( tbl_name = "{table}" and sql like '%VIRTUAL TABLE%USING FTS%' diff --git a/tests/test_utils.py b/tests/test_utils.py index e9e722b8..4b14126e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -159,7 +159,8 @@ def test_validate_sql_select_good(good_sql): utils.validate_sql_select(good_sql) -def test_detect_fts(): +@pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")]) +def test_detect_fts(open_quote, close_quote): sql = """ CREATE TABLE "Dumb_Table" ( "TreeID" INTEGER, @@ -175,9 +176,11 @@ def test_detect_fts(): "qCaretaker" TEXT ); CREATE VIEW Test_View AS SELECT * FROM Dumb_Table; - CREATE VIRTUAL TABLE "Street_Tree_List_fts" USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content="Street_Tree_List"); + CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close}); CREATE VIRTUAL TABLE r USING rtree(a, b, c); - """ + """.format( + open=open_quote, close=close_quote + ) conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert None is utils.detect_fts(conn, "Dumb_Table") From 0fc8afde0eb5ef677f4ac31601540d6168c8208d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Sep 2019 17:40:53 -0700 Subject: [PATCH 25/81] Changelog for 0.29.3 release --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 69f799de..26d0f75c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_29_3: + +0.29.3 (2019-09-02) +------------------- + +- Fixed implementation of CodeMirror on database page (`#560 `__) +- Documentation typo fixes - thanks, Min ho Kim (`#561 `__) +- Mechanism for detecting if a table has FTS enabled now works if the table name used alternative escaping mechanisms (`#570 `__) - for compatibility with `a recent change to sqlite-utils `__. + .. _v0_29_2: 0.29.2 (2019-07-13) From a314b761866d250c16f1ff6dd682010cf4181eb4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Oct 2019 08:32:47 -0700 Subject: [PATCH 26/81] Added /-/threads debugging page --- datasette/app.py | 13 +++++++++++++ docs/introspection.rst | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 501d1467..41a4eb37 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -457,6 +457,15 @@ class Datasette: for p in ps ] + def threads(self): + threads = list(threading.enumerate()) + return { + "num_threads": len(threads), + "threads": [ + {"name": t.name, "ident": t.ident, "daemon": t.daemon} for t in threads + ], + } + def table_metadata(self, database, table): "Fetch table-specific metadata." return ( @@ -621,6 +630,10 @@ class Datasette: JsonDataView.as_asgi(self, "config.json", lambda: self._config), r"/-/config(?P(\.json)?)$", ) + add_route( + JsonDataView.as_asgi(self, "threads.json", self.threads), + r"/-/threads(?P(\.json)?)$", + ) add_route( JsonDataView.as_asgi(self, "databases.json", self.connected_databases), r"/-/databases(?P(\.json)?)$", diff --git a/docs/introspection.rst b/docs/introspection.rst index e514ddf5..02552bab 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -90,6 +90,8 @@ Shows the :ref:`config` options for this instance of Datasette. `Config example "sql_time_limit_ms": 1000 } +.. _JsonDataView_databases: + /-/databases ------------ @@ -105,3 +107,26 @@ Shows currently attached databases. `Databases example `_:: + + { + "num_threads": 2, + "threads": [ + { + "daemon": false, + "ident": 4759197120, + "name": "MainThread" + }, + { + "daemon": true, + "ident": 123145319682048, + "name": "Thread-1" + }, + ] + } From fffd69ec031b83f46680f192ba57a27f0d1f0b8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 6 Oct 2019 10:23:58 -0700 Subject: [PATCH 27/81] Allow EXPLAIN WITH... - closes #583 --- datasette/utils/__init__.py | 2 ++ tests/test_utils.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 115c4cbe..449217b5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -167,6 +167,8 @@ allowed_sql_res = [ re.compile(r"^explain select\b"), re.compile(r"^explain query plan select\b"), re.compile(r"^with\b"), + re.compile(r"^explain with\b"), + re.compile(r"^explain query plan with\b"), ] disallawed_sql_res = [(re.compile("pragma"), "Statement may not contain PRAGMA")] diff --git a/tests/test_utils.py b/tests/test_utils.py index 4b14126e..28b0d0e1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -151,8 +151,12 @@ def test_validate_sql_select_bad(bad_sql): "select count(*) from airports", "select foo from bar", "select 1 + 1", + "explain select 1 + 1", + "explain query plan select 1 + 1", "SELECT\nblah FROM foo", "WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", + "explain WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", + "explain query plan WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", ], ) def test_validate_sql_select_good(good_sql): From af2e6a5cf186a7200d76cb67ac30fa59cc24d84e Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 14 Oct 2019 05:46:12 +0200 Subject: [PATCH 28/81] Button to format SQL, closes #136 SQL code will be formatted on page load, and can additionally be formatted by clicking the "Format SQL" button. Thanks, @rixx! --- datasette/static/app.css | 20 +++++++++++++++----- datasette/static/sql-formatter-2.3.3.min.js | 5 +++++ datasette/templates/_codemirror.html | 1 + datasette/templates/_codemirror_foot.html | 21 ++++++++++++++++++++- datasette/templates/database.html | 5 ++++- datasette/templates/query.html | 7 +++++-- 6 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 datasette/static/sql-formatter-2.3.3.min.js diff --git a/datasette/static/app.css b/datasette/static/app.css index 80eda0e3..34eb122c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -166,22 +166,32 @@ form input[type=search] { width: 95%; } } -form input[type=submit] { - color: #fff; - background-color: #007bff; - border-color: #007bff; +form input[type=submit], form button[type=button] { font-weight: 400; cursor: pointer; text-align: center; vertical-align: middle; - border: 1px solid blue; + border-width: 1px; + border-style: solid; padding: .5em 0.8em; font-size: 0.9rem; line-height: 1; border-radius: .25rem; +} + +form input[type=submit] { + color: #fff; + background-color: #007bff; + border-color: #007bff; -webkit-appearance: button; } +form button[type=button] { + color: #007bff; + background-color: #fff; + border-color: #007bff; +} + .filter-row { margin-bottom: 0.6em; } diff --git a/datasette/static/sql-formatter-2.3.3.min.js b/datasette/static/sql-formatter-2.3.3.min.js new file mode 100644 index 00000000..8f5c7e9f --- /dev/null +++ b/datasette/static/sql-formatter-2.3.3.min.js @@ -0,0 +1,5 @@ +/* sql-formatter 2.3.3 + * Copyright (c) 2016-present ZeroTurnaround LLC. (MIT Licensed) + */ +!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t():"function"==typeof define&&define.amd?define([],t):"object"==typeof exports?exports.sqlFormatter=t():e.sqlFormatter=t()}(this,function(){return function(e){function t(n){if(E[n])return E[n].exports;var r=E[n]={exports:{},id:n,loaded:!1};return e[n].call(r.exports,r,r.exports,t),r.loaded=!0,r.exports}var E={};return t.m=e,t.c=E,t.p="",t(0)}([function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}t.__esModule=!0;var r=E(24),o=n(r),T=E(25),R=n(T),N=E(26),i=n(N),A=E(27),I=n(A);t.default={format:function(e,t){switch(t=t||{},t.language){case"db2":return new o.default(t).format(e);case"n1ql":return new R.default(t).format(e);case"pl/sql":return new i.default(t).format(e);case"sql":case void 0:return new I.default(t).format(e);default:throw Error("Unsupported SQL dialect: "+t.language)}}},e.exports=t.default},function(e,t,E){var n=E(12),r="object"==typeof self&&self&&self.Object===Object&&self,o=n||r||Function("return this")();e.exports=o},function(e,t,E){function n(e){return null==e?void 0===e?N:R:i&&i in Object(e)?o(e):T(e)}var r=E(9),o=E(48),T=E(57),R="[object Null]",N="[object Undefined]",i=r?r.toStringTag:void 0;e.exports=n},function(e,t,E){function n(e,t){var E=o(e,t);return r(E)?E:void 0}var r=E(39),o=E(50);e.exports=n},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(74),T=n(o),R=E(8),N=n(R),i=E(21),A=n(i),I=E(22),O=n(I),u=E(23),S=n(u),a=function(){function e(t,E){r(this,e),this.cfg=t||{},this.indentation=new A.default(this.cfg.indent),this.inlineBlock=new O.default,this.params=new S.default(this.cfg.params),this.tokenizer=E,this.previousReservedWord={},this.tokens=[],this.index=0}return e.prototype.format=function(e){this.tokens=this.tokenizer.tokenize(e);var t=this.getFormattedQueryFromTokens();return t.trim()},e.prototype.getFormattedQueryFromTokens=function(){var e=this,t="";return this.tokens.forEach(function(E,n){e.index=n,E.type===N.default.WHITESPACE||(E.type===N.default.LINE_COMMENT?t=e.formatLineComment(E,t):E.type===N.default.BLOCK_COMMENT?t=e.formatBlockComment(E,t):E.type===N.default.RESERVED_TOPLEVEL?(t=e.formatToplevelReservedWord(E,t),e.previousReservedWord=E):E.type===N.default.RESERVED_NEWLINE?(t=e.formatNewlineReservedWord(E,t),e.previousReservedWord=E):E.type===N.default.RESERVED?(t=e.formatWithSpaces(E,t),e.previousReservedWord=E):t=E.type===N.default.OPEN_PAREN?e.formatOpeningParentheses(E,t):E.type===N.default.CLOSE_PAREN?e.formatClosingParentheses(E,t):E.type===N.default.PLACEHOLDER?e.formatPlaceholder(E,t):","===E.value?e.formatComma(E,t):":"===E.value?e.formatWithSpaceAfter(E,t):"."===E.value?e.formatWithoutSpaces(E,t):";"===E.value?e.formatQuerySeparator(E,t):e.formatWithSpaces(E,t))}),t},e.prototype.formatLineComment=function(e,t){return this.addNewline(t+e.value)},e.prototype.formatBlockComment=function(e,t){return this.addNewline(this.addNewline(t)+this.indentComment(e.value))},e.prototype.indentComment=function(e){return e.replace(/\n/g,"\n"+this.indentation.getIndent())},e.prototype.formatToplevelReservedWord=function(e,t){return this.indentation.decreaseTopLevel(),t=this.addNewline(t),this.indentation.increaseToplevel(),t+=this.equalizeWhitespace(e.value),this.addNewline(t)},e.prototype.formatNewlineReservedWord=function(e,t){return this.addNewline(t)+this.equalizeWhitespace(e.value)+" "},e.prototype.equalizeWhitespace=function(e){return e.replace(/\s+/g," ")},e.prototype.formatOpeningParentheses=function(e,t){var E=[N.default.WHITESPACE,N.default.OPEN_PAREN,N.default.LINE_COMMENT];return E.includes(this.previousToken().type)||(t=(0,T.default)(t)),t+=e.value,this.inlineBlock.beginIfPossible(this.tokens,this.index),this.inlineBlock.isActive()||(this.indentation.increaseBlockLevel(),t=this.addNewline(t)),t},e.prototype.formatClosingParentheses=function(e,t){return this.inlineBlock.isActive()?(this.inlineBlock.end(),this.formatWithSpaceAfter(e,t)):(this.indentation.decreaseBlockLevel(),this.formatWithSpaces(e,this.addNewline(t)))},e.prototype.formatPlaceholder=function(e,t){return t+this.params.get(e)+" "},e.prototype.formatComma=function(e,t){return t=this.trimTrailingWhitespace(t)+e.value+" ",this.inlineBlock.isActive()?t:/^LIMIT$/i.test(this.previousReservedWord.value)?t:this.addNewline(t)},e.prototype.formatWithSpaceAfter=function(e,t){return this.trimTrailingWhitespace(t)+e.value+" "},e.prototype.formatWithoutSpaces=function(e,t){return this.trimTrailingWhitespace(t)+e.value},e.prototype.formatWithSpaces=function(e,t){return t+e.value+" "},e.prototype.formatQuerySeparator=function(e,t){return this.trimTrailingWhitespace(t)+e.value+"\n"},e.prototype.addNewline=function(e){return(0,T.default)(e)+"\n"+this.indentation.getIndent()},e.prototype.trimTrailingWhitespace=function(e){return this.previousNonWhitespaceToken().type===N.default.LINE_COMMENT?(0,T.default)(e)+"\n":(0,T.default)(e)},e.prototype.previousNonWhitespaceToken=function(){for(var e=1;this.previousToken(e).type===N.default.WHITESPACE;)e++;return this.previousToken(e)},e.prototype.previousToken=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;return this.tokens[this.index-e]||{}},e}();t.default=a,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(66),T=n(o),R=E(63),N=n(R),i=E(8),A=n(i),I=function(){function e(t){r(this,e),this.WHITESPACE_REGEX=/^(\s+)/,this.NUMBER_REGEX=/^((-\s*)?[0-9]+(\.[0-9]+)?|0x[0-9a-fA-F]+|0b[01]+)\b/,this.OPERATOR_REGEX=/^(!=|<>|==|<=|>=|!<|!>|\|\||::|->>|->|~~\*|~~|!~~\*|!~~|~\*|!~\*|!~|.)/,this.BLOCK_COMMENT_REGEX=/^(\/\*[^]*?(?:\*\/|$))/,this.LINE_COMMENT_REGEX=this.createLineCommentRegex(t.lineCommentTypes),this.RESERVED_TOPLEVEL_REGEX=this.createReservedWordRegex(t.reservedToplevelWords),this.RESERVED_NEWLINE_REGEX=this.createReservedWordRegex(t.reservedNewlineWords),this.RESERVED_PLAIN_REGEX=this.createReservedWordRegex(t.reservedWords),this.WORD_REGEX=this.createWordRegex(t.specialWordChars),this.STRING_REGEX=this.createStringRegex(t.stringTypes),this.OPEN_PAREN_REGEX=this.createParenRegex(t.openParens),this.CLOSE_PAREN_REGEX=this.createParenRegex(t.closeParens),this.INDEXED_PLACEHOLDER_REGEX=this.createPlaceholderRegex(t.indexedPlaceholderTypes,"[0-9]*"),this.IDENT_NAMED_PLACEHOLDER_REGEX=this.createPlaceholderRegex(t.namedPlaceholderTypes,"[a-zA-Z0-9._$]+"),this.STRING_NAMED_PLACEHOLDER_REGEX=this.createPlaceholderRegex(t.namedPlaceholderTypes,this.createStringPattern(t.stringTypes))}return e.prototype.createLineCommentRegex=function(e){return RegExp("^((?:"+e.map(function(e){return(0,N.default)(e)}).join("|")+").*?(?:\n|$))")},e.prototype.createReservedWordRegex=function(e){var t=e.join("|").replace(/ /g,"\\s+");return RegExp("^("+t+")\\b","i")},e.prototype.createWordRegex=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];return RegExp("^([\\w"+e.join("")+"]+)")},e.prototype.createStringRegex=function(e){return RegExp("^("+this.createStringPattern(e)+")")},e.prototype.createStringPattern=function(e){var t={"``":"((`[^`]*($|`))+)","[]":"((\\[[^\\]]*($|\\]))(\\][^\\]]*($|\\]))*)",'""':'(("[^"\\\\]*(?:\\\\.[^"\\\\]*)*("|$))+)',"''":"(('[^'\\\\]*(?:\\\\.[^'\\\\]*)*('|$))+)","N''":"((N'[^N'\\\\]*(?:\\\\.[^N'\\\\]*)*('|$))+)"};return e.map(function(e){return t[e]}).join("|")},e.prototype.createParenRegex=function(e){var t=this;return RegExp("^("+e.map(function(e){return t.escapeParen(e)}).join("|")+")","i")},e.prototype.escapeParen=function(e){return 1===e.length?(0,N.default)(e):"\\b"+e+"\\b"},e.prototype.createPlaceholderRegex=function(e,t){if((0,T.default)(e))return!1;var E=e.map(N.default).join("|");return RegExp("^((?:"+E+")(?:"+t+"))")},e.prototype.tokenize=function(e){for(var t=[],E=void 0;e.length;)E=this.getNextToken(e,E),e=e.substring(E.value.length),t.push(E);return t},e.prototype.getNextToken=function(e,t){return this.getWhitespaceToken(e)||this.getCommentToken(e)||this.getStringToken(e)||this.getOpenParenToken(e)||this.getCloseParenToken(e)||this.getPlaceholderToken(e)||this.getNumberToken(e)||this.getReservedWordToken(e,t)||this.getWordToken(e)||this.getOperatorToken(e)},e.prototype.getWhitespaceToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.WHITESPACE,regex:this.WHITESPACE_REGEX})},e.prototype.getCommentToken=function(e){return this.getLineCommentToken(e)||this.getBlockCommentToken(e)},e.prototype.getLineCommentToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.LINE_COMMENT,regex:this.LINE_COMMENT_REGEX})},e.prototype.getBlockCommentToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.BLOCK_COMMENT,regex:this.BLOCK_COMMENT_REGEX})},e.prototype.getStringToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.STRING,regex:this.STRING_REGEX})},e.prototype.getOpenParenToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.OPEN_PAREN,regex:this.OPEN_PAREN_REGEX})},e.prototype.getCloseParenToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.CLOSE_PAREN,regex:this.CLOSE_PAREN_REGEX})},e.prototype.getPlaceholderToken=function(e){return this.getIdentNamedPlaceholderToken(e)||this.getStringNamedPlaceholderToken(e)||this.getIndexedPlaceholderToken(e)},e.prototype.getIdentNamedPlaceholderToken=function(e){return this.getPlaceholderTokenWithKey({input:e,regex:this.IDENT_NAMED_PLACEHOLDER_REGEX,parseKey:function(e){return e.slice(1)}})},e.prototype.getStringNamedPlaceholderToken=function(e){var t=this;return this.getPlaceholderTokenWithKey({input:e,regex:this.STRING_NAMED_PLACEHOLDER_REGEX,parseKey:function(e){return t.getEscapedPlaceholderKey({key:e.slice(2,-1),quoteChar:e.slice(-1)})}})},e.prototype.getIndexedPlaceholderToken=function(e){return this.getPlaceholderTokenWithKey({input:e,regex:this.INDEXED_PLACEHOLDER_REGEX,parseKey:function(e){return e.slice(1)}})},e.prototype.getPlaceholderTokenWithKey=function(e){var t=e.input,E=e.regex,n=e.parseKey,r=this.getTokenOnFirstMatch({input:t,regex:E,type:A.default.PLACEHOLDER});return r&&(r.key=n(r.value)),r},e.prototype.getEscapedPlaceholderKey=function(e){var t=e.key,E=e.quoteChar;return t.replace(RegExp((0,N.default)("\\")+E,"g"),E)},e.prototype.getNumberToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.NUMBER,regex:this.NUMBER_REGEX})},e.prototype.getOperatorToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.OPERATOR,regex:this.OPERATOR_REGEX})},e.prototype.getReservedWordToken=function(e,t){if(!t||!t.value||"."!==t.value)return this.getToplevelReservedToken(e)||this.getNewlineReservedToken(e)||this.getPlainReservedToken(e)},e.prototype.getToplevelReservedToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.RESERVED_TOPLEVEL,regex:this.RESERVED_TOPLEVEL_REGEX})},e.prototype.getNewlineReservedToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.RESERVED_NEWLINE,regex:this.RESERVED_NEWLINE_REGEX})},e.prototype.getPlainReservedToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.RESERVED,regex:this.RESERVED_PLAIN_REGEX})},e.prototype.getWordToken=function(e){return this.getTokenOnFirstMatch({input:e,type:A.default.WORD,regex:this.WORD_REGEX})},e.prototype.getTokenOnFirstMatch=function(e){var t=e.input,E=e.type,n=e.regex,r=t.match(n);if(r)return{type:E,value:r[1]}},e}();t.default=I,e.exports=t.default},function(e,t){function E(e){var t=typeof e;return null!=e&&("object"==t||"function"==t)}e.exports=E},function(e,t){function E(e){return null!=e&&"object"==typeof e}e.exports=E},function(e,t){"use strict";t.__esModule=!0,t.default={WHITESPACE:"whitespace",WORD:"word",STRING:"string",RESERVED:"reserved",RESERVED_TOPLEVEL:"reserved-toplevel",RESERVED_NEWLINE:"reserved-newline",OPERATOR:"operator",OPEN_PAREN:"open-paren",CLOSE_PAREN:"close-paren",LINE_COMMENT:"line-comment",BLOCK_COMMENT:"block-comment",NUMBER:"number",PLACEHOLDER:"placeholder"},e.exports=t.default},function(e,t,E){var n=E(1),r=n.Symbol;e.exports=r},function(e,t,E){function n(e){return null==e?"":r(e)}var r=E(11);e.exports=n},function(e,t,E){function n(e){if("string"==typeof e)return e;if(T(e))return o(e,n)+"";if(R(e))return A?A.call(e):"";var t=e+"";return"0"==t&&1/e==-N?"-0":t}var r=E(9),o=E(33),T=E(15),R=E(19),N=1/0,i=r?r.prototype:void 0,A=i?i.toString:void 0;e.exports=n},function(e,t){(function(t){var E="object"==typeof t&&t&&t.Object===Object&&t;e.exports=E}).call(t,function(){return this}())},function(e,t){function E(e){var t=e&&e.constructor,E="function"==typeof t&&t.prototype||n;return e===E}var n=Object.prototype;e.exports=E},function(e,t){function E(e){if(null!=e){try{return r.call(e)}catch(e){}try{return e+""}catch(e){}}return""}var n=Function.prototype,r=n.toString;e.exports=E},function(e,t){var E=Array.isArray;e.exports=E},function(e,t,E){function n(e){return null!=e&&o(e.length)&&!r(e)}var r=E(17),o=E(18);e.exports=n},function(e,t,E){function n(e){if(!o(e))return!1;var t=r(e);return t==R||t==N||t==T||t==i}var r=E(2),o=E(6),T="[object AsyncFunction]",R="[object Function]",N="[object GeneratorFunction]",i="[object Proxy]";e.exports=n},function(e,t){function E(e){return"number"==typeof e&&e>-1&&e%1==0&&n>=e}var n=9007199254740991;e.exports=E},function(e,t,E){function n(e){return"symbol"==typeof e||o(e)&&r(e)==T}var r=E(2),o=E(7),T="[object Symbol]";e.exports=n},function(e,t){e.exports=function(e){return e.webpackPolyfill||(e.deprecate=function(){},e.paths=[],e.children=[],e.webpackPolyfill=1),e}},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(69),T=n(o),R=E(68),N=n(R),i="top-level",A="block-level",I=function(){function e(t){r(this,e),this.indent=t||" ",this.indentTypes=[]}return e.prototype.getIndent=function(){return(0,T.default)(this.indent,this.indentTypes.length)},e.prototype.increaseToplevel=function(){this.indentTypes.push(i)},e.prototype.increaseBlockLevel=function(){this.indentTypes.push(A)},e.prototype.decreaseTopLevel=function(){(0,N.default)(this.indentTypes)===i&&this.indentTypes.pop()},e.prototype.decreaseBlockLevel=function(){for(;this.indentTypes.length>0;){var e=this.indentTypes.pop();if(e!==i)break}},e}();t.default=I,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(8),T=n(o),R=50,N=function(){function e(){r(this,e),this.level=0}return e.prototype.beginIfPossible=function(e,t){0===this.level&&this.isInlineBlock(e,t)?this.level=1:this.level>0?this.level++:this.level=0},e.prototype.end=function(){this.level--},e.prototype.isActive=function(){return this.level>0},e.prototype.isInlineBlock=function(e,t){for(var E=0,n=0,r=t;e.length>r;r++){var o=e[r];if(E+=o.value.length,E>R)return!1;if(o.type===T.default.OPEN_PAREN)n++;else if(o.type===T.default.CLOSE_PAREN&&(n--,0===n))return!0;if(this.isForbiddenToken(o))return!1}return!1},e.prototype.isForbiddenToken=function(e){var t=e.type,E=e.value;return t===T.default.RESERVED_TOPLEVEL||t===T.default.RESERVED_NEWLINE||t===T.default.COMMENT||t===T.default.BLOCK_COMMENT||";"===E},e}();t.default=N,e.exports=t.default},function(e,t){"use strict";function E(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var n=function(){function e(t){E(this,e),this.params=t,this.index=0}return e.prototype.get=function(e){var t=e.key,E=e.value;return this.params?t?this.params[t]:this.params[this.index++]:E},e}();t.default=n,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(4),T=n(o),R=E(5),N=n(R),i=["ABS","ACTIVATE","ALIAS","ALL","ALLOCATE","ALLOW","ALTER","ANY","ARE","ARRAY","AS","ASC","ASENSITIVE","ASSOCIATE","ASUTIME","ASYMMETRIC","AT","ATOMIC","ATTRIBUTES","AUDIT","AUTHORIZATION","AUX","AUXILIARY","AVG","BEFORE","BEGIN","BETWEEN","BIGINT","BINARY","BLOB","BOOLEAN","BOTH","BUFFERPOOL","BY","CACHE","CALL","CALLED","CAPTURE","CARDINALITY","CASCADED","CASE","CAST","CCSID","CEIL","CEILING","CHAR","CHARACTER","CHARACTER_LENGTH","CHAR_LENGTH","CHECK","CLOB","CLONE","CLOSE","CLUSTER","COALESCE","COLLATE","COLLECT","COLLECTION","COLLID","COLUMN","COMMENT","COMMIT","CONCAT","CONDITION","CONNECT","CONNECTION","CONSTRAINT","CONTAINS","CONTINUE","CONVERT","CORR","CORRESPONDING","COUNT","COUNT_BIG","COVAR_POP","COVAR_SAMP","CREATE","CROSS","CUBE","CUME_DIST","CURRENT","CURRENT_DATE","CURRENT_DEFAULT_TRANSFORM_GROUP","CURRENT_LC_CTYPE","CURRENT_PATH","CURRENT_ROLE","CURRENT_SCHEMA","CURRENT_SERVER","CURRENT_TIME","CURRENT_TIMESTAMP","CURRENT_TIMEZONE","CURRENT_TRANSFORM_GROUP_FOR_TYPE","CURRENT_USER","CURSOR","CYCLE","DATA","DATABASE","DATAPARTITIONNAME","DATAPARTITIONNUM","DATE","DAY","DAYS","DB2GENERAL","DB2GENRL","DB2SQL","DBINFO","DBPARTITIONNAME","DBPARTITIONNUM","DEALLOCATE","DEC","DECIMAL","DECLARE","DEFAULT","DEFAULTS","DEFINITION","DELETE","DENSERANK","DENSE_RANK","DEREF","DESCRIBE","DESCRIPTOR","DETERMINISTIC","DIAGNOSTICS","DISABLE","DISALLOW","DISCONNECT","DISTINCT","DO","DOCUMENT","DOUBLE","DROP","DSSIZE","DYNAMIC","EACH","EDITPROC","ELEMENT","ELSE","ELSEIF","ENABLE","ENCODING","ENCRYPTION","END","END-EXEC","ENDING","ERASE","ESCAPE","EVERY","EXCEPTION","EXCLUDING","EXCLUSIVE","EXEC","EXECUTE","EXISTS","EXIT","EXP","EXPLAIN","EXTENDED","EXTERNAL","EXTRACT","FALSE","FENCED","FETCH","FIELDPROC","FILE","FILTER","FINAL","FIRST","FLOAT","FLOOR","FOR","FOREIGN","FREE","FULL","FUNCTION","FUSION","GENERAL","GENERATED","GET","GLOBAL","GOTO","GRANT","GRAPHIC","GROUP","GROUPING","HANDLER","HASH","HASHED_VALUE","HINT","HOLD","HOUR","HOURS","IDENTITY","IF","IMMEDIATE","IN","INCLUDING","INCLUSIVE","INCREMENT","INDEX","INDICATOR","INDICATORS","INF","INFINITY","INHERIT","INNER","INOUT","INSENSITIVE","INSERT","INT","INTEGER","INTEGRITY","INTERSECTION","INTERVAL","INTO","IS","ISOBID","ISOLATION","ITERATE","JAR","JAVA","KEEP","KEY","LABEL","LANGUAGE","LARGE","LATERAL","LC_CTYPE","LEADING","LEAVE","LEFT","LIKE","LINKTYPE","LN","LOCAL","LOCALDATE","LOCALE","LOCALTIME","LOCALTIMESTAMP","LOCATOR","LOCATORS","LOCK","LOCKMAX","LOCKSIZE","LONG","LOOP","LOWER","MAINTAINED","MATCH","MATERIALIZED","MAX","MAXVALUE","MEMBER","MERGE","METHOD","MICROSECOND","MICROSECONDS","MIN","MINUTE","MINUTES","MINVALUE","MOD","MODE","MODIFIES","MODULE","MONTH","MONTHS","MULTISET","NAN","NATIONAL","NATURAL","NCHAR","NCLOB","NEW","NEW_TABLE","NEXTVAL","NO","NOCACHE","NOCYCLE","NODENAME","NODENUMBER","NOMAXVALUE","NOMINVALUE","NONE","NOORDER","NORMALIZE","NORMALIZED","NOT","NULL","NULLIF","NULLS","NUMERIC","NUMPARTS","OBID","OCTET_LENGTH","OF","OFFSET","OLD","OLD_TABLE","ON","ONLY","OPEN","OPTIMIZATION","OPTIMIZE","OPTION","ORDER","OUT","OUTER","OVER","OVERLAPS","OVERLAY","OVERRIDING","PACKAGE","PADDED","PAGESIZE","PARAMETER","PART","PARTITION","PARTITIONED","PARTITIONING","PARTITIONS","PASSWORD","PATH","PERCENTILE_CONT","PERCENTILE_DISC","PERCENT_RANK","PIECESIZE","PLAN","POSITION","POWER","PRECISION","PREPARE","PREVVAL","PRIMARY","PRIQTY","PRIVILEGES","PROCEDURE","PROGRAM","PSID","PUBLIC","QUERY","QUERYNO","RANGE","RANK","READ","READS","REAL","RECOVERY","RECURSIVE","REF","REFERENCES","REFERENCING","REFRESH","REGR_AVGX","REGR_AVGY","REGR_COUNT","REGR_INTERCEPT","REGR_R2","REGR_SLOPE","REGR_SXX","REGR_SXY","REGR_SYY","RELEASE","RENAME","REPEAT","RESET","RESIGNAL","RESTART","RESTRICT","RESULT","RESULT_SET_LOCATOR","RETURN","RETURNS","REVOKE","RIGHT","ROLE","ROLLBACK","ROLLUP","ROUND_CEILING","ROUND_DOWN","ROUND_FLOOR","ROUND_HALF_DOWN","ROUND_HALF_EVEN","ROUND_HALF_UP","ROUND_UP","ROUTINE","ROW","ROWNUMBER","ROWS","ROWSET","ROW_NUMBER","RRN","RUN","SAVEPOINT","SCHEMA","SCOPE","SCRATCHPAD","SCROLL","SEARCH","SECOND","SECONDS","SECQTY","SECURITY","SENSITIVE","SEQUENCE","SESSION","SESSION_USER","SIGNAL","SIMILAR","SIMPLE","SMALLINT","SNAN","SOME","SOURCE","SPECIFIC","SPECIFICTYPE","SQL","SQLEXCEPTION","SQLID","SQLSTATE","SQLWARNING","SQRT","STACKED","STANDARD","START","STARTING","STATEMENT","STATIC","STATMENT","STAY","STDDEV_POP","STDDEV_SAMP","STOGROUP","STORES","STYLE","SUBMULTISET","SUBSTRING","SUM","SUMMARY","SYMMETRIC","SYNONYM","SYSFUN","SYSIBM","SYSPROC","SYSTEM","SYSTEM_USER","TABLE","TABLESAMPLE","TABLESPACE","THEN","TIME","TIMESTAMP","TIMEZONE_HOUR","TIMEZONE_MINUTE","TO","TRAILING","TRANSACTION","TRANSLATE","TRANSLATION","TREAT","TRIGGER","TRIM","TRUE","TRUNCATE","TYPE","UESCAPE","UNDO","UNIQUE","UNKNOWN","UNNEST","UNTIL","UPPER","USAGE","USER","USING","VALIDPROC","VALUE","VARCHAR","VARIABLE","VARIANT","VARYING","VAR_POP","VAR_SAMP","VCAT","VERSION","VIEW","VOLATILE","VOLUMES","WHEN","WHENEVER","WHILE","WIDTH_BUCKET","WINDOW","WITH","WITHIN","WITHOUT","WLM","WRITE","XMLELEMENT","XMLEXISTS","XMLNAMESPACES","YEAR","YEARS"],A=["ADD","AFTER","ALTER COLUMN","ALTER TABLE","DELETE FROM","EXCEPT","FETCH FIRST","FROM","GROUP BY","GO","HAVING","INSERT INTO","INTERSECT","LIMIT","ORDER BY","SELECT","SET CURRENT SCHEMA","SET SCHEMA","SET","UNION ALL","UPDATE","VALUES","WHERE"],I=["AND","CROSS JOIN","INNER JOIN","JOIN","LEFT JOIN","LEFT OUTER JOIN","OR","OUTER JOIN","RIGHT JOIN","RIGHT OUTER JOIN"],O=void 0,u=function(){function e(t){r(this,e),this.cfg=t}return e.prototype.format=function(e){return O||(O=new N.default({reservedWords:i,reservedToplevelWords:A,reservedNewlineWords:I,stringTypes:['""',"''","``","[]"],openParens:["("],closeParens:[")"],indexedPlaceholderTypes:["?"],namedPlaceholderTypes:[":"],lineCommentTypes:["--"],specialWordChars:["#","@"]})),new T.default(this.cfg,O).format(e)},e}();t.default=u,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(4),T=n(o),R=E(5),N=n(R),i=["ALL","ALTER","ANALYZE","AND","ANY","ARRAY","AS","ASC","BEGIN","BETWEEN","BINARY","BOOLEAN","BREAK","BUCKET","BUILD","BY","CALL","CASE","CAST","CLUSTER","COLLATE","COLLECTION","COMMIT","CONNECT","CONTINUE","CORRELATE","COVER","CREATE","DATABASE","DATASET","DATASTORE","DECLARE","DECREMENT","DELETE","DERIVED","DESC","DESCRIBE","DISTINCT","DO","DROP","EACH","ELEMENT","ELSE","END","EVERY","EXCEPT","EXCLUDE","EXECUTE","EXISTS","EXPLAIN","FALSE","FETCH","FIRST","FLATTEN","FOR","FORCE","FROM","FUNCTION","GRANT","GROUP","GSI","HAVING","IF","IGNORE","ILIKE","IN","INCLUDE","INCREMENT","INDEX","INFER","INLINE","INNER","INSERT","INTERSECT","INTO","IS","JOIN","KEY","KEYS","KEYSPACE","KNOWN","LAST","LEFT","LET","LETTING","LIKE","LIMIT","LSM","MAP","MAPPING","MATCHED","MATERIALIZED","MERGE","MINUS","MISSING","NAMESPACE","NEST","NOT","NULL","NUMBER","OBJECT","OFFSET","ON","OPTION","OR","ORDER","OUTER","OVER","PARSE","PARTITION","PASSWORD","PATH","POOL","PREPARE","PRIMARY","PRIVATE","PRIVILEGE","PROCEDURE","PUBLIC","RAW","REALM","REDUCE","RENAME","RETURN","RETURNING","REVOKE","RIGHT","ROLE","ROLLBACK","SATISFIES","SCHEMA","SELECT","SELF","SEMI","SET","SHOW","SOME","START","STATISTICS","STRING","SYSTEM","THEN","TO","TRANSACTION","TRIGGER","TRUE","TRUNCATE","UNDER","UNION","UNIQUE","UNKNOWN","UNNEST","UNSET","UPDATE","UPSERT","USE","USER","USING","VALIDATE","VALUE","VALUED","VALUES","VIA","VIEW","WHEN","WHERE","WHILE","WITH","WITHIN","WORK","XOR"],A=["DELETE FROM","EXCEPT ALL","EXCEPT","EXPLAIN DELETE FROM","EXPLAIN UPDATE","EXPLAIN UPSERT","FROM","GROUP BY","HAVING","INFER","INSERT INTO","INTERSECT ALL","INTERSECT","LET","LIMIT","MERGE","NEST","ORDER BY","PREPARE","SELECT","SET CURRENT SCHEMA","SET SCHEMA","SET","UNION ALL","UNION","UNNEST","UPDATE","UPSERT","USE KEYS","VALUES","WHERE"],I=["AND","INNER JOIN","JOIN","LEFT JOIN","LEFT OUTER JOIN","OR","OUTER JOIN","RIGHT JOIN","RIGHT OUTER JOIN","XOR"],O=void 0,u=function(){function e(t){r(this,e),this.cfg=t}return e.prototype.format=function(e){return O||(O=new N.default({reservedWords:i,reservedToplevelWords:A,reservedNewlineWords:I,stringTypes:['""',"''","``"],openParens:["(","[","{"],closeParens:[")","]","}"],namedPlaceholderTypes:["$"],lineCommentTypes:["#","--"]})),new T.default(this.cfg,O).format(e)},e}();t.default=u,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(4),T=n(o),R=E(5),N=n(R),i=["A","ACCESSIBLE","AGENT","AGGREGATE","ALL","ALTER","ANY","ARRAY","AS","ASC","AT","ATTRIBUTE","AUTHID","AVG","BETWEEN","BFILE_BASE","BINARY_INTEGER","BINARY","BLOB_BASE","BLOCK","BODY","BOOLEAN","BOTH","BOUND","BULK","BY","BYTE","C","CALL","CALLING","CASCADE","CASE","CHAR_BASE","CHAR","CHARACTER","CHARSET","CHARSETFORM","CHARSETID","CHECK","CLOB_BASE","CLONE","CLOSE","CLUSTER","CLUSTERS","COALESCE","COLAUTH","COLLECT","COLUMNS","COMMENT","COMMIT","COMMITTED","COMPILED","COMPRESS","CONNECT","CONSTANT","CONSTRUCTOR","CONTEXT","CONTINUE","CONVERT","COUNT","CRASH","CREATE","CREDENTIAL","CURRENT","CURRVAL","CURSOR","CUSTOMDATUM","DANGLING","DATA","DATE_BASE","DATE","DAY","DECIMAL","DEFAULT","DEFINE","DELETE","DESC","DETERMINISTIC","DIRECTORY","DISTINCT","DO","DOUBLE","DROP","DURATION","ELEMENT","ELSIF","EMPTY","ESCAPE","EXCEPTIONS","EXCLUSIVE","EXECUTE","EXISTS","EXIT","EXTENDS","EXTERNAL","EXTRACT","FALSE","FETCH","FINAL","FIRST","FIXED","FLOAT","FOR","FORALL","FORCE","FROM","FUNCTION","GENERAL","GOTO","GRANT","GROUP","HASH","HEAP","HIDDEN","HOUR","IDENTIFIED","IF","IMMEDIATE","IN","INCLUDING","INDEX","INDEXES","INDICATOR","INDICES","INFINITE","INSTANTIABLE","INT","INTEGER","INTERFACE","INTERVAL","INTO","INVALIDATE","IS","ISOLATION","JAVA","LANGUAGE","LARGE","LEADING","LENGTH","LEVEL","LIBRARY","LIKE","LIKE2","LIKE4","LIKEC","LIMITED","LOCAL","LOCK","LONG","MAP","MAX","MAXLEN","MEMBER","MERGE","MIN","MINUS","MINUTE","MLSLABEL","MOD","MODE","MONTH","MULTISET","NAME","NAN","NATIONAL","NATIVE","NATURAL","NATURALN","NCHAR","NEW","NEXTVAL","NOCOMPRESS","NOCOPY","NOT","NOWAIT","NULL","NULLIF","NUMBER_BASE","NUMBER","OBJECT","OCICOLL","OCIDATE","OCIDATETIME","OCIDURATION","OCIINTERVAL","OCILOBLOCATOR","OCINUMBER","OCIRAW","OCIREF","OCIREFCURSOR","OCIROWID","OCISTRING","OCITYPE","OF","OLD","ON","ONLY","OPAQUE","OPEN","OPERATOR","OPTION","ORACLE","ORADATA","ORDER","ORGANIZATION","ORLANY","ORLVARY","OTHERS","OUT","OVERLAPS","OVERRIDING","PACKAGE","PARALLEL_ENABLE","PARAMETER","PARAMETERS","PARENT","PARTITION","PASCAL","PCTFREE","PIPE","PIPELINED","PLS_INTEGER","PLUGGABLE","POSITIVE","POSITIVEN","PRAGMA","PRECISION","PRIOR","PRIVATE","PROCEDURE","PUBLIC","RAISE","RANGE","RAW","READ","REAL","RECORD","REF","REFERENCE","RELEASE","RELIES_ON","REM","REMAINDER","RENAME","RESOURCE","RESULT_CACHE","RESULT","RETURN","RETURNING","REVERSE","REVOKE","ROLLBACK","ROW","ROWID","ROWNUM","ROWTYPE","SAMPLE","SAVE","SAVEPOINT","SB1","SB2","SB4","SECOND","SEGMENT","SELF","SEPARATE","SEQUENCE","SERIALIZABLE","SHARE","SHORT","SIZE_T","SIZE","SMALLINT","SOME","SPACE","SPARSE","SQL","SQLCODE","SQLDATA","SQLERRM","SQLNAME","SQLSTATE","STANDARD","START","STATIC","STDDEV","STORED","STRING","STRUCT","STYLE","SUBMULTISET","SUBPARTITION","SUBSTITUTABLE","SUBTYPE","SUCCESSFUL","SUM","SYNONYM","SYSDATE","TABAUTH","TABLE","TDO","THE","THEN","TIME","TIMESTAMP","TIMEZONE_ABBR","TIMEZONE_HOUR","TIMEZONE_MINUTE","TIMEZONE_REGION","TO","TRAILING","TRANSACTION","TRANSACTIONAL","TRIGGER","TRUE","TRUSTED","TYPE","UB1","UB2","UB4","UID","UNDER","UNIQUE","UNPLUG","UNSIGNED","UNTRUSTED","USE","USER","USING","VALIDATE","VALIST","VALUE","VARCHAR","VARCHAR2","VARIABLE","VARIANCE","VARRAY","VARYING","VIEW","VIEWS","VOID","WHENEVER","WHILE","WITH","WORK","WRAPPED","WRITE","YEAR","ZONE"],A=["ADD","ALTER COLUMN","ALTER TABLE","BEGIN","CONNECT BY","DECLARE","DELETE FROM","DELETE","END","EXCEPT","EXCEPTION","FETCH FIRST","FROM","GROUP BY","HAVING","INSERT INTO","INSERT","INTERSECT","LIMIT","LOOP","MODIFY","ORDER BY","SELECT","SET CURRENT SCHEMA","SET SCHEMA","SET","START WITH","UNION ALL","UNION","UPDATE","VALUES","WHERE"],I=["AND","CROSS APPLY","CROSS JOIN","ELSE","END","INNER JOIN","JOIN","LEFT JOIN","LEFT OUTER JOIN","OR","OUTER APPLY","OUTER JOIN","RIGHT JOIN","RIGHT OUTER JOIN","WHEN","XOR"],O=void 0,u=function(){function e(t){r(this,e),this.cfg=t}return e.prototype.format=function(e){return O||(O=new N.default({reservedWords:i,reservedToplevelWords:A,reservedNewlineWords:I,stringTypes:['""',"N''","''","``"],openParens:["(","CASE"],closeParens:[")","END"],indexedPlaceholderTypes:["?"],namedPlaceholderTypes:[":"],lineCommentTypes:["--"],specialWordChars:["_","$","#",".","@"]})),new T.default(this.cfg,O).format(e)},e}();t.default=u,e.exports=t.default},function(e,t,E){"use strict";function n(e){return e&&e.__esModule?e:{default:e}}function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}t.__esModule=!0;var o=E(4),T=n(o),R=E(5),N=n(R),i=["ACCESSIBLE","ACTION","AGAINST","AGGREGATE","ALGORITHM","ALL","ALTER","ANALYSE","ANALYZE","AS","ASC","AUTOCOMMIT","AUTO_INCREMENT","BACKUP","BEGIN","BETWEEN","BINLOG","BOTH","CASCADE","CASE","CHANGE","CHANGED","CHARACTER SET","CHARSET","CHECK","CHECKSUM","COLLATE","COLLATION","COLUMN","COLUMNS","COMMENT","COMMIT","COMMITTED","COMPRESSED","CONCURRENT","CONSTRAINT","CONTAINS","CONVERT","CREATE","CROSS","CURRENT_TIMESTAMP","DATABASE","DATABASES","DAY","DAY_HOUR","DAY_MINUTE","DAY_SECOND","DEFAULT","DEFINER","DELAYED","DELETE","DESC","DESCRIBE","DETERMINISTIC","DISTINCT","DISTINCTROW","DIV","DO","DROP","DUMPFILE","DUPLICATE","DYNAMIC","ELSE","ENCLOSED","END","ENGINE","ENGINES","ENGINE_TYPE","ESCAPE","ESCAPED","EVENTS","EXEC","EXECUTE","EXISTS","EXPLAIN","EXTENDED","FAST","FETCH","FIELDS","FILE","FIRST","FIXED","FLUSH","FOR","FORCE","FOREIGN","FULL","FULLTEXT","FUNCTION","GLOBAL","GRANT","GRANTS","GROUP_CONCAT","HEAP","HIGH_PRIORITY","HOSTS","HOUR","HOUR_MINUTE","HOUR_SECOND","IDENTIFIED","IF","IFNULL","IGNORE","IN","INDEX","INDEXES","INFILE","INSERT","INSERT_ID","INSERT_METHOD","INTERVAL","INTO","INVOKER","IS","ISOLATION","KEY","KEYS","KILL","LAST_INSERT_ID","LEADING","LEVEL","LIKE","LINEAR","LINES","LOAD","LOCAL","LOCK","LOCKS","LOGS","LOW_PRIORITY","MARIA","MASTER","MASTER_CONNECT_RETRY","MASTER_HOST","MASTER_LOG_FILE","MATCH","MAX_CONNECTIONS_PER_HOUR","MAX_QUERIES_PER_HOUR","MAX_ROWS","MAX_UPDATES_PER_HOUR","MAX_USER_CONNECTIONS","MEDIUM","MERGE","MINUTE","MINUTE_SECOND","MIN_ROWS","MODE","MODIFY","MONTH","MRG_MYISAM","MYISAM","NAMES","NATURAL","NOT","NOW()","NULL","OFFSET","ON DELETE","ON UPDATE","ON","ONLY","OPEN","OPTIMIZE","OPTION","OPTIONALLY","OUTFILE","PACK_KEYS","PAGE","PARTIAL","PARTITION","PARTITIONS","PASSWORD","PRIMARY","PRIVILEGES","PROCEDURE","PROCESS","PROCESSLIST","PURGE","QUICK","RAID0","RAID_CHUNKS","RAID_CHUNKSIZE","RAID_TYPE","RANGE","READ","READ_ONLY","READ_WRITE","REFERENCES","REGEXP","RELOAD","RENAME","REPAIR","REPEATABLE","REPLACE","REPLICATION","RESET","RESTORE","RESTRICT","RETURN","RETURNS","REVOKE","RLIKE","ROLLBACK","ROW","ROWS","ROW_FORMAT","SECOND","SECURITY","SEPARATOR","SERIALIZABLE","SESSION","SHARE","SHOW","SHUTDOWN","SLAVE","SONAME","SOUNDS","SQL","SQL_AUTO_IS_NULL","SQL_BIG_RESULT","SQL_BIG_SELECTS","SQL_BIG_TABLES","SQL_BUFFER_RESULT","SQL_CACHE","SQL_CALC_FOUND_ROWS","SQL_LOG_BIN","SQL_LOG_OFF","SQL_LOG_UPDATE","SQL_LOW_PRIORITY_UPDATES","SQL_MAX_JOIN_SIZE","SQL_NO_CACHE","SQL_QUOTE_SHOW_CREATE","SQL_SAFE_UPDATES","SQL_SELECT_LIMIT","SQL_SLAVE_SKIP_COUNTER","SQL_SMALL_RESULT","SQL_WARNINGS","START","STARTING","STATUS","STOP","STORAGE","STRAIGHT_JOIN","STRING","STRIPED","SUPER","TABLE","TABLES","TEMPORARY","TERMINATED","THEN","TO","TRAILING","TRANSACTIONAL","TRUE","TRUNCATE","TYPE","TYPES","UNCOMMITTED","UNIQUE","UNLOCK","UNSIGNED","USAGE","USE","USING","VARIABLES","VIEW","WHEN","WITH","WORK","WRITE","YEAR_MONTH"],A=["ADD","AFTER","ALTER COLUMN","ALTER TABLE","DELETE FROM","EXCEPT","FETCH FIRST","FROM","GROUP BY","GO","HAVING","INSERT INTO","INSERT","INTERSECT","LIMIT","MODIFY","ORDER BY","SELECT","SET CURRENT SCHEMA","SET SCHEMA","SET","UNION ALL","UNION","UPDATE","VALUES","WHERE"],I=["AND","CROSS APPLY","CROSS JOIN","ELSE","INNER JOIN","JOIN","LEFT JOIN","LEFT OUTER JOIN","OR","OUTER APPLY","OUTER JOIN","RIGHT JOIN","RIGHT OUTER JOIN","WHEN","XOR"],O=void 0,u=function(){ +function e(t){r(this,e),this.cfg=t}return e.prototype.format=function(e){return O||(O=new N.default({reservedWords:i,reservedToplevelWords:A,reservedNewlineWords:I,stringTypes:['""',"N''","''","``","[]"],openParens:["(","CASE"],closeParens:[")","END"],indexedPlaceholderTypes:["?"],namedPlaceholderTypes:["@",":"],lineCommentTypes:["#","--"]})),new T.default(this.cfg,O).format(e)},e}();t.default=u,e.exports=t.default},function(e,t,E){var n=E(3),r=E(1),o=n(r,"DataView");e.exports=o},function(e,t,E){var n=E(3),r=E(1),o=n(r,"Map");e.exports=o},function(e,t,E){var n=E(3),r=E(1),o=n(r,"Promise");e.exports=o},function(e,t,E){var n=E(3),r=E(1),o=n(r,"Set");e.exports=o},function(e,t,E){var n=E(3),r=E(1),o=n(r,"WeakMap");e.exports=o},function(e,t){function E(e,t){for(var E=-1,n=null==e?0:e.length,r=Array(n);++Et||t>n)return E;do t%2&&(E+=e),t=r(t/2),t&&(e+=e);while(t);return E}var n=9007199254740991,r=Math.floor;e.exports=E},function(e,t){function E(e,t,E){var n=-1,r=e.length;0>t&&(t=-t>r?0:r+t),E=E>r?r:E,0>E&&(E+=r),r=t>E?0:E-t>>>0,t>>>=0;for(var o=Array(r);++nE?r(e,t,E):e}var r=E(43);e.exports=n},function(e,t,E){function n(e,t){for(var E=e.length;E--&&r(t,e[E],0)>-1;);return E}var r=E(36);e.exports=n},function(e,t,E){var n=E(1),r=n["__core-js_shared__"];e.exports=r},function(e,t,E){function n(e){var t=T.call(e,N),E=e[N];try{e[N]=void 0;var n=!0}catch(e){}var r=R.call(e);return n&&(t?e[N]=E:delete e[N]),r}var r=E(9),o=Object.prototype,T=o.hasOwnProperty,R=o.toString,N=r?r.toStringTag:void 0;e.exports=n},function(e,t,E){var n=E(28),r=E(29),o=E(30),T=E(31),R=E(32),N=E(2),i=E(14),A="[object Map]",I="[object Object]",O="[object Promise]",u="[object Set]",S="[object WeakMap]",a="[object DataView]",s=i(n),L=i(r),c=i(o),C=i(T),f=i(R),p=N;(n&&p(new n(new ArrayBuffer(1)))!=a||r&&p(new r)!=A||o&&p(o.resolve())!=O||T&&p(new T)!=u||R&&p(new R)!=S)&&(p=function(e){var t=N(e),E=t==I?e.constructor:void 0,n=E?i(E):"";if(n)switch(n){case s:return a;case L:return A;case c:return O;case C:return u;case f:return S}return t}),e.exports=p},function(e,t){function E(e,t){return null==e?void 0:e[t]}e.exports=E},function(e,t){function E(e){return A.test(e)}var n="\\ud800-\\udfff",r="\\u0300-\\u036f",o="\\ufe20-\\ufe2f",T="\\u20d0-\\u20ff",R=r+o+T,N="\\ufe0e\\ufe0f",i="\\u200d",A=RegExp("["+i+n+R+N+"]");e.exports=E},function(e,t){function E(e,t){var E=typeof e;return t=null==t?n:t,!!t&&("number"==E||"symbol"!=E&&r.test(e))&&e>-1&&e%1==0&&t>e}var n=9007199254740991,r=/^(?:0|[1-9]\d*)$/;e.exports=E},function(e,t,E){function n(e,t,E){if(!R(E))return!1;var n=typeof t;return!!("number"==n?o(E)&&T(t,E.length):"string"==n&&t in E)&&r(E[t],e)}var r=E(62),o=E(16),T=E(52),R=E(6);e.exports=n},function(e,t,E){function n(e){return!!o&&o in e}var r=E(47),o=function(){var e=/[^.]+$/.exec(r&&r.keys&&r.keys.IE_PROTO||"");return e?"Symbol(src)_1."+e:""}();e.exports=n},function(e,t,E){var n=E(58),r=n(Object.keys,Object);e.exports=r},function(e,t,E){(function(e){var n=E(12),r="object"==typeof t&&t&&!t.nodeType&&t,o=r&&"object"==typeof e&&e&&!e.nodeType&&e,T=o&&o.exports===r,R=T&&n.process,N=function(){try{var e=o&&o.require&&o.require("util").types;return e?e:R&&R.binding&&R.binding("util")}catch(e){}}();e.exports=N}).call(t,E(20)(e))},function(e,t){function E(e){return r.call(e)}var n=Object.prototype,r=n.toString;e.exports=E},function(e,t){function E(e,t){return function(E){return e(t(E))}}e.exports=E},function(e,t){function E(e,t,E){for(var n=E-1,r=e.length;++ne?-1:1;return t*T}return e===e?e:0}var r=E(73),o=1/0,T=1.7976931348623157e308;e.exports=n},function(e,t,E){function n(e){var t=r(e),E=t%1;return t===t?E?t-E:t:0}var r=E(71);e.exports=n},function(e,t,E){function n(e){if("number"==typeof e)return e;if(o(e))return T;if(r(e)){var t="function"==typeof e.valueOf?e.valueOf():e;e=r(t)?t+"":t}if("string"!=typeof e)return 0===e?e:+e;e=e.replace(R,"");var E=i.test(e);return E||A.test(e)?I(e.slice(2),E?2:8):N.test(e)?T:+e}var r=E(6),o=E(19),T=NaN,R=/^\s+|\s+$/g,N=/^[-+]0x[0-9a-f]+$/i,i=/^0b[01]+$/i,A=/^0o[0-7]+$/i,I=parseInt;e.exports=n},function(e,t,E){function n(e,t,E){if(e=N(e),e&&(E||void 0===t))return e.replace(i,"");if(!e||!(t=r(t)))return e;var n=R(e),A=T(n,R(t))+1;return o(n,0,A).join("")}var r=E(11),o=E(45),T=E(46),R=E(60),N=E(10),i=/\s+$/;e.exports=n}])}); diff --git a/datasette/templates/_codemirror.html b/datasette/templates/_codemirror.html index 237d6907..17342be5 100644 --- a/datasette/templates/_codemirror.html +++ b/datasette/templates/_codemirror.html @@ -1,3 +1,4 @@ + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 4b55bf8d..9bc6d97f 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,5 +1,18 @@ diff --git a/datasette/templates/database.html b/datasette/templates/database.html index a934f336..a0d0fcf6 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -26,7 +26,10 @@

Custom SQL query

-

+

+ + +

{% endif %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7c6c59f3..34fa78a5 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -37,7 +37,7 @@ {% if editable and config.allow_sql %}

{% else %} - 
{% if query %}{{ query.sql }}{% endif %}
+ 
{% if query %}{{ query.sql }}{% endif %}
{% endif %} {% else %} @@ -49,7 +49,10 @@

{% endfor %} {% endif %} -

+

+ + +

{% if display_rows %} From 908fc3999e06f3ccd3bb8ad0539490bbc7809748 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 14 Oct 2019 05:52:33 +0200 Subject: [PATCH 29/81] Sort databases on homepage by argument order - #591 Closes #585 - thanks, @rixx! --- datasette/app.py | 2 +- datasette/views/index.py | 2 -- tests/test_html.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 41a4eb37..935b1730 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -159,7 +159,7 @@ class Datasette: self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files - self.databases = {} + self.databases = collections.OrderedDict() self.inspect_data = inspect_data for file in self.files: path = file diff --git a/datasette/views/index.py b/datasette/views/index.py index fddb04d9..f2e5f774 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -97,8 +97,6 @@ class IndexView(BaseView): } ) - databases.sort(key=lambda database: database["name"]) - if as_format: headers = {} if self.ds.cors: diff --git a/tests/test_html.py b/tests/test_html.py index 0a6df984..ec7765f6 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -26,11 +26,11 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": "/extra_database", "text": "extra_database"}, {"href": "/fixtures", "text": "fixtures"}, + {"href": "/extra_database", "text": "extra_database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables - h2 = soup.select("h2")[0] + h2 = soup.select("h2")[1] assert "extra_database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] assert ( From 12cec411cae73ba7211429da12cd32c551fe17b1 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 14 Oct 2019 05:53:21 +0200 Subject: [PATCH 30/81] Display metadata footer on custom SQL queries (#589) Closes #408 - thanks, @rixx! --- datasette/views/database.py | 10 ++++++---- tests/test_html.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 78af19c5..31d6af59 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,12 +10,17 @@ class DatabaseView(DataView): name = "database" async def data(self, request, database, hash, default_labels=False, _size=None): + metadata = (self.ds.metadata("databases") or {}).get(database, {}) + self.ds.update_with_inherited_metadata(metadata) + if request.args.get("sql"): if not self.ds.config("allow_sql"): raise DatasetteError("sql= is not allowed", status=400) sql = request.raw_args.pop("sql") validate_sql_select(sql) - return await self.custom_sql(request, database, hash, sql, _size=_size) + return await self.custom_sql( + request, database, hash, sql, _size=_size, metadata=metadata + ) db = self.ds.databases[database] @@ -24,9 +29,6 @@ class DatabaseView(DataView): hidden_table_names = set(await db.hidden_table_names()) all_foreign_keys = await db.get_all_foreign_keys() - metadata = (self.ds.metadata("databases") or {}).get(database, {}) - self.ds.update_with_inherited_metadata(metadata) - tables = [] for table in table_counts: table_columns = await db.table_columns(table) diff --git a/tests/test_html.py b/tests/test_html.py index ec7765f6..0bb1c163 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -737,6 +737,18 @@ def test_database_metadata(app_client): assert_footer_links(soup) +def test_database_metadata_with_custom_sql(app_client): + response = app_client.get("/fixtures?sql=select+*+from+simple_primary_key") + assert response.status == 200 + soup = Soup(response.body, "html.parser") + # Page title should be the default + assert "fixtures" == soup.find("h1").text + # Description should be custom + assert "Custom SQL query returning" in soup.find("h3").text + # The source/license should be inherited + assert_footer_links(soup) + + def test_table_metadata(app_client): response = app_client.get("/fixtures/simple_primary_key") assert response.status == 200 From 9366d0bf191daccee6093c54ed51a2855d129cd8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Oct 2019 15:29:16 -0700 Subject: [PATCH 31/81] Add Python versions badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 59a6649e..a4db6611 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) +[![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=master)](https://travis-ci.org/simonw/datasette) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/master/LICENSE) From 3e864b1625f3142e6ff084f9b41247f2f9f60f80 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Oct 2019 14:51:45 -0700 Subject: [PATCH 32/81] Use --platform=managed for publish cloudrun, closes #587 --- datasette/publish/cloudrun.py | 2 +- tests/test_publish_cloudrun.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 32c9cd2a..c2d77746 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -110,7 +110,7 @@ def publish_subcommand(publish): image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( - "gcloud beta run deploy --allow-unauthenticated --image {}{}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}{}".format( image_id, " {}".format(service) if service else "" ), shell=True, diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 1e9bb830..481ac04d 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -40,7 +40,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): [ mock.call("gcloud builds submit --tag {}".format(tag), shell=True), mock.call( - "gcloud beta run deploy --allow-unauthenticated --image {}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}".format( tag ), shell=True, From b6ad1fdc7068cb8248787843e7438d1f19fa2e3a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Oct 2019 22:23:01 -0700 Subject: [PATCH 33/81] Fixed bug returning non-ascii characters in CSV, closes #584 --- datasette/utils/asgi.py | 2 +- tests/test_csv.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index eaf3428d..bafcfb4a 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -217,7 +217,7 @@ class AsgiWriter: await self.send( { "type": "http.response.body", - "body": chunk.encode("latin-1"), + "body": chunk.encode("utf-8"), "more_body": True, } ) diff --git a/tests/test_csv.py b/tests/test_csv.py index c3cdc241..1d5d2df2 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -80,6 +80,15 @@ def test_table_csv_download(app_client): assert expected_disposition == response.headers["Content-Disposition"] +def test_csv_with_non_ascii_characters(app_client): + response = app_client.get( + "/fixtures.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number" + ) + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.body.decode("utf8") + + def test_max_csv_mb(app_client_csv_max_mb_one): response = app_client_csv_max_mb_one.get( "/fixtures.csv?sql=select+randomblob(10000)+" From b647b5efc29300f715ba656e41b0591f342938e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 15:51:07 -0700 Subject: [PATCH 34/81] Fix for /foo v.s. /foo-bar issue, closes #597 Pull request #599 --- datasette/views/base.py | 16 ++++++++-------- tests/fixtures.py | 7 +++++++ tests/test_api.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index db1d69d9..219630af 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -193,14 +193,14 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None - if "-" in db_name: - # Might be name-and-hash, or might just be - # a name with a hyphen in it - name, hash = db_name.rsplit("-", 1) - if name not in self.ds.databases: - # Try the whole name - name = db_name - hash = None + if db_name not in self.ds.databases and "-" in db_name: + # No matching DB found, maybe it's a name-hash? + name_bit, hash_bit = db_name.rsplit("-", 1) + if name_bit not in self.ds.databases: + raise NotFound("Database not found: {}".format(name)) + else: + name = name_bit + hash = hash_bit else: name = db_name # Verify the hash diff --git a/tests/fixtures.py b/tests/fixtures.py index dac28dc0..a4c32f36 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -178,6 +178,13 @@ def app_client_two_attached_databases(): ) +@pytest.fixture(scope="session") +def app_client_conflicting_database_names(): + yield from make_app_client( + extra_databases={"foo.db": EXTRA_DATABASE_SQL, "foo-bar.db": EXTRA_DATABASE_SQL} + ) + + @pytest.fixture(scope="session") def app_client_two_attached_databases_one_immutable(): yield from make_app_client( diff --git a/tests/test_api.py b/tests/test_api.py index cc00b780..826c00f3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -7,6 +7,7 @@ from .fixtures import ( # noqa app_client_larger_cache_size, app_client_returned_rows_matches_page_size, app_client_two_attached_databases_one_immutable, + app_client_conflicting_database_names, app_client_with_cors, app_client_with_dot, generate_compound_rows, @@ -1652,3 +1653,20 @@ def test_cors(app_client_with_cors, path, status_code): response = app_client_with_cors.get(path) assert response.status == status_code assert "*" == response.headers["Access-Control-Allow-Origin"] + + +def test_common_prefix_database_names(app_client_conflicting_database_names): + # https://github.com/simonw/datasette/issues/597 + assert ["fixtures", "foo", "foo-bar"] == [ + d["name"] + for d in json.loads( + app_client_conflicting_database_names.get("/-/databases.json").body.decode( + "utf8" + ) + ) + ] + for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): + data = json.loads( + app_client_conflicting_database_names.get(path).body.decode("utf8") + ) + assert db_name == data["database"] From e877b1cb12076946fdbec7ca2fbfbfc75c1c2a28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 16:56:44 -0700 Subject: [PATCH 35/81] Don't auto-format SQL on page load (#601) Closes #600 --- datasette/templates/_codemirror_foot.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 9bc6d97f..9aba61ab 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -6,12 +6,6 @@ window.onload = () => { if (sqlFormat && !readOnly) { sqlFormat.hidden = false; } - if (readOnly) { - readOnly.innerHTML = sqlFormatter.format(readOnly.innerHTML); - } - if (sqlInput) { - sqlInput.value = sqlFormatter.format(sqlInput.value); - } var editor = CodeMirror.fromTextArea(sqlInput, { lineNumbers: true, mode: "text/x-sql", From debea4f971c180af64e16b83be98d830e9dee54f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 18:05:47 -0700 Subject: [PATCH 36/81] Release 0.30 --- docs/changelog.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26d0f75c..e8dafa35 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_30: + +0.30 (2019-10-18) +----------------- + +- Added ``/-/threads`` debugging page +- Allow ``EXPLAIN WITH...`` (`#583 `__) +- Button to format SQL - thanks, Tobias Kunze (`#136 `__) +- Sort databases on homepage by argument order - thanks, Tobias Kunze (`#585 `__) +- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (`#589 `__) +- Use ``--platform=managed`` for ``publish cloudrun`` (`#587 `__) +- Fixed bug returning non-ASCII characters in CSV (`#584 `__) +- Fix for ``/foo`` v.s. ``/foo-bar`` bug (`#601 `__) + .. _v0_29_3: 0.29.3 (2019-09-02) From 8050f9e1ece9afd0236ad38c6458c12a4ad917e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Oct 2019 18:08:04 -0700 Subject: [PATCH 37/81] Update news in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a4db6611..5894017e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. From f4c0830529a9513a83437a9e1550bbe27ebc5c64 Mon Sep 17 00:00:00 2001 From: chris48s Date: Mon, 21 Oct 2019 03:03:08 +0100 Subject: [PATCH 38/81] Always pop as_format off args dict (#603) Closes #563. Thanks, @chris48s --- datasette/views/base.py | 2 ++ tests/test_api.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/datasette/views/base.py b/datasette/views/base.py index 219630af..348f0c03 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -362,6 +362,8 @@ class DataView(BaseView): _format = request.args.get("_format", None) if not _format: _format = (args.pop("as_format", None) or "").lstrip(".") + else: + args.pop("as_format", None) if "table_and_format" in args: db = self.ds.databases[database] diff --git a/tests/test_api.py b/tests/test_api.py index 826c00f3..a734b8de 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1107,6 +1107,15 @@ def test_row(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] +def test_row_format_in_querystring(app_client): + # regression test for https://github.com/simonw/datasette/issues/563 + response = app_client.get( + "/fixtures/simple_primary_key/1?_format=json&_shape=objects" + ) + assert response.status == 200 + assert [{"id": "1", "content": "hello"}] == response.json["rows"] + + def test_row_strange_table_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" From 5dd4d2b2d3abcfd507a6df47e7c2fbad3c552fd8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:49:01 -0700 Subject: [PATCH 39/81] Update to latest black (#609) --- datasette/views/base.py | 9 ++++++--- datasette/views/table.py | 7 ++++--- setup.py | 2 +- tests/test_api.py | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 348f0c03..1568b084 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -257,9 +257,12 @@ class DataView(BaseView): assert NotImplemented async def get(self, request, db_name, **kwargs): - database, hash, correct_hash_provided, should_redirect = await self.resolve_db_name( - request, db_name, **kwargs - ) + ( + database, + hash, + correct_hash_provided, + should_redirect, + ) = await self.resolve_db_name(request, db_name, **kwargs) if should_redirect: return self.redirect(request, should_redirect, remove_args={"_hash"}) diff --git a/datasette/views/table.py b/datasette/views/table.py index 8ba3abe4..e0362e53 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -586,9 +586,10 @@ class TableView(RowTableShared): ) for facet in facet_instances: - instance_facet_results, instance_facets_timed_out = ( - await facet.facet_results() - ) + ( + instance_facet_results, + instance_facets_timed_out, + ) = await facet.facet_results() facet_results.update(instance_facet_results) facets_timed_out.extend(instance_facets_timed_out) diff --git a/setup.py b/setup.py index cbe545a1..9ae56306 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def get_version(): # Only install black on Python 3.6 or higher maybe_black = [] if sys.version_info > (3, 6): - maybe_black = ["black"] + maybe_black = ["black~=19.10b0"] setup( name="datasette", diff --git a/tests/test_api.py b/tests/test_api.py index a734b8de..4ea95e84 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1245,7 +1245,7 @@ def test_config_json(app_client): def test_page_size_matching_max_returned_rows( - app_client_returned_rows_matches_page_size + app_client_returned_rows_matches_page_size, ): fetched = [] path = "/fixtures/no_primary_key.json" From e2c390500e6782aa476a7edc05c46cf907875a6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:49:26 -0700 Subject: [PATCH 40/81] Persist _where= in hidden fields, closes #604 --- datasette/views/table.py | 3 +++ tests/test_html.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/datasette/views/table.py b/datasette/views/table.py index e0362e53..652ce994 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -721,6 +721,9 @@ class TableView(RowTableShared): for arg in ("_fts_table", "_fts_pk"): if arg in special_args: form_hidden_args.append((arg, special_args[arg])) + if request.args["_where"]: + for where_text in request.args["_where"]: + form_hidden_args.append(("_where", where_text)) return { "supports_search": bool(fts_table), "search": search or "", diff --git a/tests/test_html.py b/tests/test_html.py index 0bb1c163..aa628dec 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -955,6 +955,12 @@ def test_extra_where_clauses(app_client): "/fixtures/facetable?_where=city_id%3D1", "/fixtures/facetable?_where=neighborhood%3D%27Dogpatch%27", ] == hrefs + # These should also be persisted as hidden fields + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [("_where", "neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [ + (hidden["name"], hidden["value"]) for hidden in hiddens + ] def test_binary_data_display(app_client): From f5f6cbe03cbf05737d848f44779372b5daa79a25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 11:56:04 -0700 Subject: [PATCH 41/81] Release 0.30.1 --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8dafa35..8ac32c45 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _v0_30_1: + +0.30.1 (2019-10-30) +------------------- + +- Fixed bug where ``?_where=`` parameter was not persisted in hidden form fields (`#604 `__) +- Fixed bug with .JSON representation of row pages - thanks, Chris Shaw (`#603 `__) + .. _v0_30: 0.30 (2019-10-18) From 3ca290e0db03bb4747e24203c445873f74512107 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Oct 2019 12:00:21 -0700 Subject: [PATCH 42/81] Fixed dumb error --- datasette/views/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 652ce994..44b186cf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -721,7 +721,7 @@ class TableView(RowTableShared): for arg in ("_fts_table", "_fts_pk"): if arg in special_args: form_hidden_args.append((arg, special_args[arg])) - if request.args["_where"]: + if request.args.get("_where"): for where_text in request.args["_where"]: form_hidden_args.append(("_where", where_text)) return { From 937828f946238c28e77ba50e0b2e649c874560f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Oct 2019 22:39:59 -0700 Subject: [PATCH 43/81] Use distinfo.project_name for plugin name if available, closes #606 --- datasette/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 449217b5..3d28a36b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -633,6 +633,7 @@ def get_plugins(pm): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version + plugin_info["name"] = distinfo.project_name plugins.append(plugin_info) return plugins From 50287e7c6bb0987536e5515f05945721c4515e9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 12:37:46 -0700 Subject: [PATCH 44/81] Only suggest array facet for arrays of strings - closes #562 --- datasette/facets.py | 44 +++++++++++++++++++++++++++++++++----------- tests/fixtures.py | 33 +++++++++++++++++---------------- tests/test_api.py | 20 +++++++++++++++++--- tests/test_csv.py | 32 ++++++++++++++++---------------- tests/test_facets.py | 9 +++++++++ 5 files changed, 92 insertions(+), 46 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 365d9c65..9b5baaa2 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -257,6 +257,16 @@ class ColumnFacet(Facet): class ArrayFacet(Facet): type = "array" + def _is_json_array_of_strings(self, json_string): + try: + array = json.loads(json_string) + except ValueError: + return False + for item in array: + if not isinstance(item, str): + return False + return True + async def suggest(self): columns = await self.get_columns(self.sql, self.params) suggested_facets = [] @@ -282,18 +292,30 @@ class ArrayFacet(Facet): ) types = tuple(r[0] for r in results.rows) if types in (("array",), ("array", None)): - suggested_facets.append( - { - "name": column, - "type": "array", - "toggle_url": self.ds.absolute_url( - self.request, - path_with_added_args( - self.request, {"_facet_array": column} - ), - ), - } + # Now sanity check that first 100 arrays contain only strings + first_100 = await self.ds.execute( + self.database, + "select {column} from ({sql}) where {column} is not null".format( + column=escape_sqlite(column), sql=self.sql + ), + self.params, + truncate=False, + custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), + log_sql_errors=False, ) + if all(self._is_json_array_of_strings(r[0]) for r in first_100): + suggested_facets.append( + { + "name": column, + "type": "array", + "toggle_url": self.ds.absolute_url( + self.request, + path_with_added_args( + self.request, {"_facet_array": column} + ), + ), + } + ) except (QueryInterrupted, sqlite3.OperationalError): continue return suggested_facets diff --git a/tests/fixtures.py b/tests/fixtures.py index a4c32f36..93c3da9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -661,26 +661,27 @@ CREATE TABLE facetable ( city_id integer, neighborhood text, tags text, + complex_array text, FOREIGN KEY ("city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (created, planet_int, on_earth, state, city_id, neighborhood, tags) + (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array) VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]'), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]'), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]'), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]'), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]') + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]'), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]'), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]'), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]'), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]') ; CREATE TABLE binary_data ( diff --git a/tests/test_api.py b/tests/test_api.py index 4ea95e84..41557bcf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -195,6 +195,7 @@ def test_database_page(app_client): "city_id", "neighborhood", "tags", + "complex_array", ], "primary_keys": ["pk"], "count": 15, @@ -1029,15 +1030,25 @@ def test_table_filter_queries_multiple_of_same_type(app_client): def test_table_filter_json_arraycontains(app_client): response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") assert [ - [1, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Mission", '["tag1", "tag2"]'], - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]'], + [ + 1, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Mission", + '["tag1", "tag2"]', + '[{"foo": "bar"}]', + ], + [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"], ] == response.json["rows"] def test_table_filter_extra_where(app_client): response = app_client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'") assert [ - [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]'] + [2, "2019-01-14 08:00:00", 1, 1, "CA", 1, "Dogpatch", '["tag1", "tag3"]', "[]"] ] == response.json["rows"] @@ -1453,6 +1464,7 @@ def test_suggested_facets(app_client): {"name": "city_id", "querystring": "_facet=city_id"}, {"name": "neighborhood", "querystring": "_facet=neighborhood"}, {"name": "tags", "querystring": "_facet=tags"}, + {"name": "complex_array", "querystring": "_facet=complex_array"}, {"name": "created", "querystring": "_facet_date=created"}, ] if detect_json1(): @@ -1488,6 +1500,7 @@ def test_expand_labels(app_client): "city_id": {"value": 1, "label": "San Francisco"}, "neighborhood": "Dogpatch", "tags": '["tag1", "tag3"]', + "complex_array": "[]", }, "13": { "pk": 13, @@ -1498,6 +1511,7 @@ def test_expand_labels(app_client): "city_id": {"value": 3, "label": "Detroit"}, "neighborhood": "Corktown", "tags": "[]", + "complex_array": "[]", }, } == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 1d5d2df2..b148b6db 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -21,22 +21,22 @@ world ) EXPECTED_TABLE_WITH_LABELS_CSV = """ -pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags -1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]" -2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]" -3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[] -4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[] -5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[] -6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[] -7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[] -8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[] -9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[] -10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[] -11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[] -12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[] -13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[] -14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[] -15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[] +pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array +1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]" +2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[] +3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[] +4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[] +5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[] +6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[] +7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[] +8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[] +9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[] +10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[] +11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[] +12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[] +13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[] +14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[] +15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[] """.lstrip().replace( "\n", "\r\n" ) diff --git a/tests/test_facets.py b/tests/test_facets.py index 9169f666..402c155b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -23,6 +23,10 @@ async def test_column_facet_suggest(app_client): {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"}, {"name": "neighborhood", "toggle_url": "http://localhost/?_facet=neighborhood"}, {"name": "tags", "toggle_url": "http://localhost/?_facet=tags"}, + { + "name": "complex_array", + "toggle_url": "http://localhost/?_facet=complex_array", + }, ] == suggestions @@ -57,6 +61,10 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client): "name": "tags", "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=tags", }, + { + "name": "complex_array", + "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=complex_array", + }, ] == suggestions @@ -78,6 +86,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): "state", "neighborhood", "tags", + "complex_array", ] == suggestions From ba5414f16b49781261d0f41a16f2210d5fa3976f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 12:38:15 -0700 Subject: [PATCH 45/81] Only inspect first 100 records for #562 --- datasette/facets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/facets.py b/datasette/facets.py index 9b5baaa2..7f350dfe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -295,7 +295,7 @@ class ArrayFacet(Facet): # Now sanity check that first 100 arrays contain only strings first_100 = await self.ds.execute( self.database, - "select {column} from ({sql}) where {column} is not null".format( + "select {column} from ({sql}) where {column} is not null limit 100".format( column=escape_sqlite(column), sql=self.sql ), self.params, From 7152e76eda9049574643261e7a471958cc16d0b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 14:45:59 -0700 Subject: [PATCH 46/81] Don't suggest array facet if column is only [], closes #610 --- datasette/facets.py | 29 ++++++++++++++++++----------- tests/test_facets.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 7f350dfe..0c6459d6 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -293,17 +293,24 @@ class ArrayFacet(Facet): types = tuple(r[0] for r in results.rows) if types in (("array",), ("array", None)): # Now sanity check that first 100 arrays contain only strings - first_100 = await self.ds.execute( - self.database, - "select {column} from ({sql}) where {column} is not null limit 100".format( - column=escape_sqlite(column), sql=self.sql - ), - self.params, - truncate=False, - custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), - log_sql_errors=False, - ) - if all(self._is_json_array_of_strings(r[0]) for r in first_100): + first_100 = [ + v[0] + for v in await self.ds.execute( + self.database, + "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format( + column=escape_sqlite(column), sql=self.sql + ), + self.params, + truncate=False, + custom_time_limit=self.ds.config( + "facet_suggest_time_limit_ms" + ), + log_sql_errors=False, + ) + ] + if first_100 and all( + self._is_json_array_of_strings(r) for r in first_100 + ): suggested_facets.append( { "name": column, diff --git a/tests/test_facets.py b/tests/test_facets.py index 402c155b..e3dc3df3 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -215,6 +215,20 @@ async def test_array_facet_suggest(app_client): ] == suggestions +@pytest.mark.asyncio +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +async def test_array_facet_suggest_not_if_all_empty_arrays(app_client): + facet = ArrayFacet( + app_client.ds, + MockRequest("http://localhost/"), + database="fixtures", + sql="select * from facetable where tags = '[]'", + table="facetable", + ) + suggestions = await facet.suggest() + assert [] == suggestions + + @pytest.mark.asyncio @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") async def test_array_facet_results(app_client): From ffae2f0ecde1ca92e78d097665df820d3b7861e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 14:57:49 -0700 Subject: [PATCH 47/81] Better documentation of --host, closes #574 --- README.md | 25 +++++++++++++++---------- datasette/cli.py | 11 +++++++++-- docs/datasette-serve-help.txt | 7 +++++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5894017e..9f85f1ba 100644 --- a/README.md +++ b/README.md @@ -89,26 +89,31 @@ Now visiting http://localhost:8001/History/downloads will show you a web interfa ## datasette serve options - $ datasette serve --help - Usage: datasette serve [OPTIONS] [FILES]... Serve up specified SQLite database files with a web UI Options: -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT host for server, defaults to 127.0.0.1 - -p, --port INTEGER port for server, defaults to 8001 + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means + only connections from the local machine will be + allowed. Use 0.0.0.0 to listen to all IPs and + allow access from other machines. + -p, --port INTEGER Port for server, defaults to 8001 --debug Enable debug mode - useful for development - --reload Automatically reload if database or code change detected - - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: * + --reload Automatically reload if database or code change + detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * --load-extension PATH Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON file containing license/source metadata + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON file containing license/source + metadata --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files --memory Make :memory: database available --config CONFIG Set config option using configname:value datasette.readthedocs.io/en/latest/config.html diff --git a/datasette/cli.py b/datasette/cli.py index 181b281c..67c2fe71 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -230,9 +230,16 @@ def package( multiple=True, ) @click.option( - "-h", "--host", default="127.0.0.1", help="host for server, defaults to 127.0.0.1" + "-h", + "--host", + default="127.0.0.1", + help=( + "Host for server. Defaults to 127.0.0.1 which means only connections " + "from the local machine will be allowed. Use 0.0.0.0 to listen to " + "all IPs and allow access from other machines." + ), ) -@click.option("-p", "--port", default=8001, help="port for server, defaults to 8001") +@click.option("-p", "--port", default=8001, help="Port for server, defaults to 8001") @click.option( "--debug", is_flag=True, help="Enable debug mode - useful for development" ) diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 7b7c3b09..1447e84d 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -6,8 +6,11 @@ Usage: datasette serve [OPTIONS] [FILES]... Options: -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT host for server, defaults to 127.0.0.1 - -p, --port INTEGER port for server, defaults to 8001 + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means only + connections from the local machine will be allowed. Use + 0.0.0.0 to listen to all IPs and allow access from other + machines. + -p, --port INTEGER Port for server, defaults to 8001 --debug Enable debug mode - useful for development --reload Automatically reload if database or code change detected - useful for development From ed57e4f99018c1d520858f55f6eee4eb1cc2af3d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Nov 2019 15:15:10 -0700 Subject: [PATCH 48/81] Plugin static assets support both hyphens and underscores in names Closes #611 --- datasette/app.py | 13 +++++++++++-- docs/plugins.rst | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 935b1730..203e0991 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -612,8 +612,17 @@ class Datasette: # Mount any plugin static/ directories for plugin in get_plugins(pm): if plugin["static_path"]: - modpath = "/-/static-plugins/{}/(?P.*)$".format(plugin["name"]) - add_route(asgi_static(plugin["static_path"]), modpath) + add_route( + asgi_static(plugin["static_path"]), + "/-/static-plugins/{}/(?P.*)$".format(plugin["name"]), + ) + # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611 + add_route( + asgi_static(plugin["static_path"]), + "/-/static-plugins/{}/(?P.*)$".format( + plugin["name"].replace("-", "_") + ), + ) add_route( JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata), r"/-/metadata(?P(\.json)?)$", diff --git a/docs/plugins.rst b/docs/plugins.rst index 1d4f1e1a..6df7ff6a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -442,7 +442,7 @@ you have one: @hookimpl def extra_js_urls(): return [ - '/-/static-plugins/your_plugin/app.js' + '/-/static-plugins/your-plugin/app.js' ] .. _plugin_hook_publish_subcommand: From 14da70525b35e1a44cd45c19101385467057f041 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 15:29:40 -0700 Subject: [PATCH 49/81] Don't show 'None' as label for nullable foreign key, closes #406 --- datasette/views/table.py | 2 +- tests/fixtures.py | 1 + tests/test_api.py | 18 ++++++++++++++++-- tests/test_html.py | 9 +++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 44b186cf..326c11ae 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -637,7 +637,7 @@ class TableView(RowTableShared): new_row = CustomRow(columns) for column in row.keys(): value = row[column] - if (column, value) in expanded_labels: + if (column, value) in expanded_labels and value is not None: new_row[column] = { "value": value, "label": expanded_labels[(column, value)], diff --git a/tests/fixtures.py b/tests/fixtures.py index 93c3da9f..8aa44687 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -754,6 +754,7 @@ INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); INSERT INTO foreign_key_references VALUES (1, 1, 1); +INSERT INTO foreign_key_references VALUES (2, null, null); INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); INSERT INTO custom_foreign_key_label VALUES (1, 1); diff --git a/tests/test_api.py b/tests/test_api.py index 41557bcf..c6acbab1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -216,7 +216,7 @@ def test_database_page(app_client): "name": "foreign_key_references", "columns": ["pk", "foreign_key_with_label", "foreign_key_with_no_label"], "primary_keys": ["pk"], - "count": 1, + "count": 2, "hidden": False, "fts_table": None, "foreign_keys": { @@ -1519,7 +1519,7 @@ def test_expand_labels(app_client): def test_expand_label(app_client): response = app_client.get( "/fixtures/foreign_key_references.json?_shape=object" - "&_label=foreign_key_with_label" + "&_label=foreign_key_with_label&_size=1" ) assert { "1": { @@ -1693,3 +1693,17 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): app_client_conflicting_database_names.get(path).body.decode("utf8") ) assert db_name == data["database"] + + +def test_null_foreign_keys_are_not_expanded(app_client): + response = app_client.get( + "/fixtures/foreign_key_references.json?_shape=array&_labels=on" + ) + assert [ + { + "pk": "1", + "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_no_label": {"value": "1", "label": "1"}, + }, + {"pk": "2", "foreign_key_with_label": None, "foreign_key_with_no_label": None,}, + ] == response.json diff --git a/tests/test_html.py b/tests/test_html.py index aa628dec..f63e595b 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -603,7 +603,12 @@ def test_table_html_foreign_key_links(app_client): '1', 'hello\xa01', '1', - ] + ], + [ + '2', + '\xa0', + '\xa0', + ], ] assert expected == [ [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") @@ -611,7 +616,7 @@ def test_table_html_foreign_key_links(app_client): def test_table_html_disable_foreign_key_links_with_labels(app_client): - response = app_client.get("/fixtures/foreign_key_references?_labels=off") + response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") expected = [ From c3181d9a840dff7be8c990b21f5749db393a4ea0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 15:47:20 -0700 Subject: [PATCH 50/81] Release notes for 0.30.2 --- docs/changelog.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ac32c45..f4761efe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v0_30_2: + +0.30.2 (2019-11-02) +------------------- + +- ``/-/plugins`` page now uses distribution name e.g. ``datasette-cluster-map`` instead of the name of the underlying Python package (``datasette_cluster_map``) (`#606 `__) +- Array faceting is now only suggested for columns that contain arrays of strings (`#562 `__) +- Better documentation for the ``--host`` argument (`#574 `__) +- Don't show ``None`` with a broken link for the label on a nullable foreign key (`#406 `__) + .. _v0_30_1: 0.30.1 (2019-10-30) @@ -14,6 +24,7 @@ Changelog .. _v0_30: + 0.30 (2019-10-18) ----------------- @@ -82,7 +93,7 @@ Two new plugins take advantage of this hook: New plugin hook: extra_template_vars ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). +The :ref:`plugin_hook_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). Secret plugin configuration options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 2bf7ce5f517d772a16d7855a35a8a75d4456aad7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Nov 2019 16:12:46 -0700 Subject: [PATCH 51/81] Fix CSV export for nullable foreign keys, closes #612 --- datasette/views/base.py | 12 ++++++++---- tests/test_csv.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 1568b084..94945304 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -330,10 +330,14 @@ class DataView(BaseView): else: # Look for {"value": "label": } dicts and expand new_row = [] - for cell in row: - if isinstance(cell, dict): - new_row.append(cell["value"]) - new_row.append(cell["label"]) + for heading, cell in zip(data["columns"], row): + if heading in expanded_columns: + if cell is None: + new_row.extend(("", "")) + else: + assert isinstance(cell, dict) + new_row.append(cell["value"]) + new_row.append(cell["label"]) else: new_row.append(cell) await writer.writerow(new_row) diff --git a/tests/test_csv.py b/tests/test_csv.py index b148b6db..13aca489 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -41,6 +41,14 @@ pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,com "\n", "\r\n" ) +EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV = """ +pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label +1,1,hello,1,1 +2,,,, +""".lstrip().replace( + "\n", "\r\n" +) + def test_table_csv(app_client): response = app_client.get("/fixtures/simple_primary_key.csv") @@ -63,6 +71,13 @@ def test_table_csv_with_labels(app_client): assert EXPECTED_TABLE_WITH_LABELS_CSV == response.text +def test_table_csv_with_nullable_labels(app_client): + response = app_client.get("/fixtures/foreign_key_references.csv?_labels=1") + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text + + def test_custom_sql_csv(app_client): response = app_client.get( "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2" From ee330222f4c3ee66c2fe41ebc76fed56b9cb9a00 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Mon, 4 Nov 2019 03:39:55 +0100 Subject: [PATCH 52/81] Offer to format readonly SQL (#602) Following discussion in #601, this PR adds a "Format SQL" button to read-only SQL (if the SQL actually differs from the formatting result). It also removes a console error on readonly SQL queries. Thanks, @rixx! --- datasette/templates/_codemirror_foot.html | 41 ++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 9aba61ab..4019d448 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -6,21 +6,32 @@ window.onload = () => { if (sqlFormat && !readOnly) { sqlFormat.hidden = false; } - var editor = CodeMirror.fromTextArea(sqlInput, { - lineNumbers: true, - mode: "text/x-sql", - lineWrapping: true, - }); - editor.setOption("extraKeys", { - "Shift-Enter": function() { - document.getElementsByClassName("sql")[0].submit(); - }, - Tab: false - }); - if (sqlInput && sqlFormat) { - sqlFormat.addEventListener("click", ev => { - editor.setValue(sqlFormatter.format(editor.getValue())); - }) + if (sqlInput) { + var editor = CodeMirror.fromTextArea(sqlInput, { + lineNumbers: true, + mode: "text/x-sql", + lineWrapping: true, + }); + editor.setOption("extraKeys", { + "Shift-Enter": function() { + document.getElementsByClassName("sql")[0].submit(); + }, + Tab: false + }); + if (sqlFormat) { + sqlFormat.addEventListener("click", ev => { + editor.setValue(sqlFormatter.format(editor.getValue())); + }) + } + } + if (sqlFormat && readOnly) { + const formatted = sqlFormatter.format(readOnly.innerHTML); + if (formatted != readOnly.innerHTML) { + sqlFormat.hidden = false; + sqlFormat.addEventListener("click", ev => { + readOnly.innerHTML = formatted; + }) + } } } From 9db22cdf1809fb78a7b183cd2f617cd5e26efc68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Nov 2019 20:11:55 -0800 Subject: [PATCH 53/81] pk__notin= filter, closes #614 --- datasette/filters.py | 15 +++++++++++++++ docs/json_api.rst | 3 +++ tests/test_filters.py | 3 +++ 3 files changed, 21 insertions(+) diff --git a/datasette/filters.py b/datasette/filters.py index efe014ae..5897a3ed 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -77,6 +77,20 @@ class InFilter(Filter): return "{} in {}".format(column, json.dumps(self.split_value(value))) +class NotInFilter(InFilter): + key = "notin" + display = "not in" + + def where_clause(self, table, column, value, param_counter): + values = self.split_value(value) + params = [":p{}".format(param_counter + i) for i in range(len(values))] + sql = "{} not in ({})".format(escape_sqlite(column), ", ".join(params)) + return sql, values + + def human_clause(self, column, value): + return "{} not in {}".format(column, json.dumps(self.split_value(value))) + + class Filters: _filters = ( [ @@ -125,6 +139,7 @@ class Filters: TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), InFilter(), + NotInFilter(), ] + ( [ diff --git a/docs/json_api.rst b/docs/json_api.rst index 4b365e14..de70362c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -228,6 +228,9 @@ You can filter the data returned by the table based on column values using a que ``?column__in=["value","value,with,commas"]`` +``?column__notin=value1,value2,value3`` + Rows where column does not match any of the provided values. The inverse of ``__in=``. Also supports JSON arrays. + ``?column__arraycontains=value`` Works against columns that contain JSON arrays - matches if any of the values in that array match. diff --git a/tests/test_filters.py b/tests/test_filters.py index fd682cd9..8598087f 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -47,6 +47,9 @@ import pytest ["foo in (:p0, :p1)"], ["dog,cat", "cat[dog]"], ), + # Not in, and JSON array not in + ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]), + ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]), ], ) def test_build_where(args, expected_where, expected_params): From 52fa79c6075f0830ff635b81d957c64d877a05aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 Nov 2019 15:03:48 -0800 Subject: [PATCH 54/81] Use select colnames, not select * for table view - refs #615 --- datasette/views/table.py | 8 ++++++-- tests/test_api.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 326c11ae..139ff80b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -235,13 +235,17 @@ class TableView(RowTableShared): raise NotFound("Table not found: {}".format(table)) pks = await db.primary_keys(table) + table_columns = await db.table_columns(table) + + select_columns = ", ".join(escape_sqlite(t) for t in table_columns) + use_rowid = not pks and not is_view if use_rowid: - select = "rowid, *" + select = "rowid, {}".format(select_columns) order_by = "rowid" order_by_pks = "rowid" else: - select = "*" + select = select_columns order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) order_by = order_by_pks diff --git a/tests/test_api.py b/tests/test_api.py index c6acbab1..4a09b238 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -610,7 +610,8 @@ def test_table_json(app_client): assert response.status == 200 data = response.json assert ( - data["query"]["sql"] == "select * from simple_primary_key order by id limit 51" + data["query"]["sql"] + == "select id, content from simple_primary_key order by id limit 51" ) assert data["query"]["params"] == {} assert data["rows"] == [ From 931bfc66613aa3e22f8314df5c0d0758baf31f38 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Tue, 5 Nov 2019 00:16:30 +0100 Subject: [PATCH 55/81] Handle spaces in DB names (#590) Closes #503 - thanks, @rixx --- datasette/views/base.py | 3 ++- tests/fixtures.py | 4 ++-- tests/test_api.py | 19 ++++++++++++++++++- tests/test_html.py | 8 ++++---- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 94945304..062c6956 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -203,12 +203,13 @@ class DataView(BaseView): hash = hash_bit else: name = db_name - # Verify the hash + name = urllib.parse.unquote_plus(name) try: db = self.ds.databases[name] except KeyError: raise NotFound("Database not found: {}".format(name)) + # Verify the hash expected = "000" if db.hash is not None: expected = db.hash[:HASH_LENGTH] diff --git a/tests/fixtures.py b/tests/fixtures.py index 8aa44687..dcc414bf 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -174,7 +174,7 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_two_attached_databases(): yield from make_app_client( - extra_databases={"extra_database.db": EXTRA_DATABASE_SQL} + extra_databases={"extra database.db": EXTRA_DATABASE_SQL} ) @@ -188,7 +188,7 @@ def app_client_conflicting_database_names(): @pytest.fixture(scope="session") def app_client_two_attached_databases_one_immutable(): yield from make_app_client( - is_immutable=True, extra_databases={"extra_database.db": EXTRA_DATABASE_SQL} + is_immutable=True, extra_databases={"extra database.db": EXTRA_DATABASE_SQL} ) diff --git a/tests/test_api.py b/tests/test_api.py index 4a09b238..1fa8642f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,6 +6,7 @@ from .fixtures import ( # noqa app_client_shorter_time_limit, app_client_larger_cache_size, app_client_returned_rows_matches_page_size, + app_client_two_attached_databases, app_client_two_attached_databases_one_immutable, app_client_conflicting_database_names, app_client_with_cors, @@ -1188,7 +1189,7 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): databases = response.json assert 2 == len(databases) extra_database, fixtures_database = databases - assert "extra_database" == extra_database["name"] + assert "extra database" == extra_database["name"] assert None == extra_database["hash"] assert True == extra_database["is_mutable"] assert False == extra_database["is_memory"] @@ -1679,6 +1680,22 @@ def test_cors(app_client_with_cors, path, status_code): assert "*" == response.headers["Access-Control-Allow-Origin"] +@pytest.mark.parametrize( + "path", + ( + "/", + ".json", + "/searchable", + "/searchable.json", + "/searchable_view", + "/searchable_view.json", + ), +) +def test_database_with_space_in_name(app_client_two_attached_databases, path): + response = app_client_two_attached_databases.get("/extra database" + path) + assert response.status == 200 + + def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 assert ["fixtures", "foo", "foo-bar"] == [ diff --git a/tests/test_html.py b/tests/test_html.py index f63e595b..7f1af86e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -27,11 +27,11 @@ def test_homepage(app_client_two_attached_databases): # Should be two attached databases assert [ {"href": "/fixtures", "text": "fixtures"}, - {"href": "/extra_database", "text": "extra_database"}, + {"href": "/extra database", "text": "extra database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables h2 = soup.select("h2")[1] - assert "extra_database" == h2.text.strip() + assert "extra database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] assert ( "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip() @@ -41,8 +41,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": "/extra_database/searchable", "text": "searchable"}, - {"href": "/extra_database/searchable_view", "text": "searchable_view"}, + {"href": "/extra database/searchable", "text": "searchable"}, + {"href": "/extra database/searchable_view", "text": "searchable_view"}, ] == table_links From c30f07c58e410ee296b28aeabe4dc461dd40b435 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Nov 2019 21:12:55 -0800 Subject: [PATCH 56/81] Removed _group_count=col feature, closes #504 --- datasette/views/table.py | 12 ------------ docs/json_api.rst | 9 --------- 2 files changed, 21 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 139ff80b..920693d7 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -499,18 +499,6 @@ class TableView(RowTableShared): if order_by: order_by = "order by {} ".format(order_by) - # _group_count=col1&_group_count=col2 - group_count = special_args_lists.get("_group_count") or [] - if group_count: - sql = 'select {group_cols}, count(*) as "count" from {table_name} {where} group by {group_cols} order by "count" desc limit 100'.format( - group_cols=", ".join( - '"{}"'.format(group_count_col) for group_count_col in group_count - ), - table_name=escape_sqlite(table), - where=where_clause, - ) - return await self.custom_sql(request, database, hash, sql, editable=True) - extra_args = {} # Handle ?_size=500 page_size = _size or request.raw_args.get("_size") diff --git a/docs/json_api.rst b/docs/json_api.rst index de70362c..e369bee7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -321,15 +321,6 @@ Special table arguments Here's `an example `__. - -``?_group_count=COLUMN`` - Executes a SQL query that returns a count of the number of rows matching - each unique value in that column, with the most common ordered first. - -``?_group_count=COLUMN1&_group_count=column2`` - You can pass multiple ``_group_count`` columns to return counts against - unique combinations of those columns. - ``?_next=TOKEN`` Pagination by continuation token - pass the token that was returned in the ``"next"`` property by the previous page. From f9c146b893856a48afa810ebcce1714f30d0d3a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Nov 2019 16:55:44 -0800 Subject: [PATCH 57/81] Removed unused special_args_lists variable --- datasette/views/table.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 920693d7..a60a3941 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -261,12 +261,10 @@ class TableView(RowTableShared): # That's so if there is a column that starts with _ # it can still be queried using ?_col__exact=blah special_args = {} - special_args_lists = {} other_args = [] for key, value in args.items(): if key.startswith("_") and "__" not in key: special_args[key] = value[0] - special_args_lists[key] = value else: for v in value: other_args.append((key, v)) From 83fc5165ac724f69cd57d8f15cd3038e7b30f878 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Nov 2019 18:48:39 -0800 Subject: [PATCH 58/81] Improved UI for publish cloudrun, closes #608 --- datasette/publish/cloudrun.py | 39 ++++++++++++++++++++++-- tests/test_publish_cloudrun.py | 55 ++++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index c2d77746..a833a32b 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -60,6 +60,23 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() + if not service: + # Show the user their current services, then prompt for one + click.echo("Please provide a service name for this deployment\n") + click.echo("Using an existing service name will over-write it") + click.echo("") + existing_services = get_existing_services() + if existing_services: + click.echo("Your existing services:\n") + for existing_service in existing_services: + click.echo( + " {name} - created {created} - {url}".format( + **existing_service + ) + ) + click.echo("") + service = click.prompt("Service name", type=str) + extra_metadata = { "title": title, "license": license, @@ -110,8 +127,26 @@ def publish_subcommand(publish): image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( - "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}{}".format( - image_id, " {}".format(service) if service else "" + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} {}".format( + image_id, service, ), shell=True, ) + + +def get_existing_services(): + services = json.loads( + check_output( + "gcloud beta run services list --platform=managed --format json", + shell=True, + universal_newlines=True, + ) + ) + return [ + { + "name": service["metadata"]["name"], + "created": service["metadata"]["creationTimestamp"], + "url": service["status"]["address"]["url"], + } + for service in services + ] diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 481ac04d..a038b60e 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -24,6 +24,53 @@ def test_publish_cloudrun_invalid_database(mock_which): assert 'Path "woop.db" does not exist' in result.output +@mock.patch("shutil.which") +@mock.patch("datasette.publish.cloudrun.check_output") +@mock.patch("datasette.publish.cloudrun.check_call") +@mock.patch("datasette.publish.cloudrun.get_existing_services") +def test_publish_cloudrun_prompts_for_service( + mock_get_existing_services, mock_call, mock_output, mock_which +): + mock_get_existing_services.return_value = [ + {"name": "existing", "created": "2019-01-01", "url": "http://www.example.com/"} + ] + mock_output.return_value = "myproject" + mock_which.return_value = True + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" + ) + assert ( + """ +Please provide a service name for this deployment + +Using an existing service name will over-write it + +Your existing services: + + existing - created 2019-01-01 - http://www.example.com/ + +Service name: input-service +""".strip() + == result.output.strip() + ) + assert 0 == result.exit_code + tag = "gcr.io/myproject/datasette" + mock_call.assert_has_calls( + [ + mock.call("gcloud builds submit --tag {}".format(tag), shell=True), + mock.call( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( + tag + ), + shell=True, + ), + ] + ) + + @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -33,14 +80,16 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): open("test.db", "w").write("data") - result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] + ) assert 0 == result.exit_code tag = "gcr.io/{}/datasette".format(mock_output.return_value) mock_call.assert_has_calls( [ mock.call("gcloud builds submit --tag {}".format(tag), shell=True), mock.call( - "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {}".format( + "gcloud beta run deploy --allow-unauthenticated --platform=managed --image {} test".format( tag ), shell=True, @@ -65,6 +114,8 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): "publish", "cloudrun", "test.db", + "--service", + "datasette", "--plugin-secret", "datasette-auth-github", "client_id", From 9f5d19c254d1bfbd99f576dff47a6e32e01c76ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Nov 2019 18:12:20 -0800 Subject: [PATCH 59/81] Improved documentation for "publish cloudrun" --- docs/publish.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index 304be8ef..89d33085 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -43,14 +43,16 @@ You will first need to install and configure the Google Cloud CLI tools by follo You can then publish a database to Google Cloud Run using the following command:: - datasette publish cloudrun mydatabase.db + datasette publish cloudrun mydatabase.db --service=my-database + +A Cloud Run **service** is a single hosted application. The service name you specify will be used as part of the Cloud Run URL. If you deploy to a service name that you have used in the past your new deployment will replace the previous one. + +If you omit the ``--service`` option you will be asked to pick a service name interactively during the deploy. You may need to interact with prompts from the tool. Once it has finished it will output a URL like this one:: - Service [datasette] revision [datasette-00001] has been deployed - and is serving traffic at https://datasette-j7hipcg4aq-uc.a.run.app - -During the deployment the tool will prompt you for the name of your service. You can reuse an existing name to replace your previous deployment with your new version, or pick a new name to deploy to a new URL. + Service [my-service] revision [my-service-00001] has been deployed + and is serving traffic at https://my-service-j7hipcg4aq-uc.a.run.app .. literalinclude:: datasette-publish-cloudrun-help.txt @@ -90,18 +92,18 @@ Custom metadata and plugins You can define your own :ref:`metadata` and deploy that with your instance like so:: - datasette publish nowv1 mydatabase.db -m metadata.json + datasette publish cloudrun --service=my-service mydatabase.db -m metadata.json If you just want to set the title, license or source information you can do that directly using extra options to ``datasette publish``:: - datasette publish nowv1 mydatabase.db \ + datasette publish cloudrun mydatabase.db --service=my-service \ --title="Title of my database" \ --source="Where the data originated" \ --source_url="http://www.example.com/" You can also specify plugins you would like to install. For example, if you want to include the `datasette-vega `_ visualization plugin you can use the following:: - datasette publish nowv1 mydatabase.db --install=datasette-vega + datasette publish cloudrun mydatabase.db --service=my-service --install=datasette-vega If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: From 10b9d85edaaf198879344aa1c498000cfb27dff8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 8 Nov 2019 18:15:13 -0800 Subject: [PATCH 60/81] datasette-csvs on Glitch now uses sqlite-utils It previously used csvs-to-sqlite but that had heavy dependencies. See https://support.glitch.com/t/can-you-upgrade-python-to-latest-version/7980/33 --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index d0c22583..fdf7d23c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -25,7 +25,7 @@ Glitch allows you to "remix" any project to create your own copy and start editi .. image:: https://cdn.glitch.com/2703baf2-b643-4da7-ab91-7ee2a2d00b5b%2Fremix-button.svg :target: https://glitch.com/edit/#!/remix/datasette-csvs -Find a CSV file and drag it onto the Glitch file explorer panel - ``datasette-csvs`` will automatically convert it to a SQLite database (using `csvs-to-sqlite `__) and allow you to start exploring it using Datasette. +Find a CSV file and drag it onto the Glitch file explorer panel - ``datasette-csvs`` will automatically convert it to a SQLite database (using `sqlite-utils `__) and allow you to start exploring it using Datasette. If your CSV file has a ``latitude`` and ``longitude`` column you can visualize it on a map by uncommenting the ``datasette-cluster-map`` line in the ``requirements.txt`` file using the Glitch file editor. From 28c4a6db5b5e512db630d7ba6127196185de67c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 9 Nov 2019 17:29:36 -0800 Subject: [PATCH 61/81] CREATE INDEX statements on table page, closes #618 --- datasette/database.py | 13 ++++++++++++- tests/fixtures.py | 1 + tests/test_html.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index 7e6f7245..3a1cea94 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -232,7 +232,18 @@ class Database: ) if not table_definition_rows: return None - return table_definition_rows[0][0] + bits = [table_definition_rows[0][0] + ";"] + # Add on any indexes + index_rows = list( + await self.ds.execute( + self.name, + "select sql from sqlite_master where tbl_name = :n and type='index' and sql is not null", + {"n": table}, + ) + ) + for index_row in index_rows: + bits.append(index_row[0] + ";") + return "\n".join(bits) async def get_view_definition(self, view): return await self.get_table_definition(view, "view") diff --git a/tests/fixtures.py b/tests/fixtures.py index dcc414bf..87e66f99 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -514,6 +514,7 @@ CREATE TABLE compound_three_primary_keys ( content text, PRIMARY KEY (pk1, pk2, pk3) ); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); CREATE TABLE foreign_key_references ( pk varchar(30) primary key, diff --git a/tests/test_html.py b/tests/test_html.py index 7f1af86e..44627cdc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -119,6 +119,39 @@ def test_row_strange_table_name_with_url_hash(app_client_with_hash): assert response.status == 200 +@pytest.mark.parametrize( + "path,expected_definition_sql", + [ + ( + "/fixtures/facet_cities", + """ +CREATE TABLE facet_cities ( + id integer primary key, + name text +); + """.strip(), + ), + ( + "/fixtures/compound_three_primary_keys", + """ +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + """.strip(), + ), + ], +) +def test_definition_sql(path, expected_definition_sql, app_client): + response = app_client.get(path) + pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql") + assert expected_definition_sql == pre.string + + def test_table_cell_truncation(): for client in make_app_client(config={"truncate_cells_html": 5}): response = client.get("/fixtures/facetable") From 1c063fae9dba70f70244db010d55a18846640f07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 Nov 2019 19:45:34 -0800 Subject: [PATCH 62/81] Test against Python 3.8 in Travis (#623) * Test against Python 3.8 in Travis * Avoid current_task warnings in Python 3.8 --- .travis.yml | 1 + datasette/tracer.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29388bc1..a6b15b7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ dist: xenial python: - "3.6" - "3.7" + - "3.8" - "3.5" # Executed for 3.5 AND 3.5 as the first "test" stage: diff --git a/datasette/tracer.py b/datasette/tracer.py index e46a6fda..a638b140 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -9,12 +9,19 @@ tracers = {} TRACE_RESERVED_KEYS = {"type", "start", "end", "duration_ms", "traceback"} +# asyncio.current_task was introduced in Python 3.7: +for obj in (asyncio, asyncio.Task): + current_task = getattr(obj, "current_task", None) + if current_task is not None: + break + + def get_task_id(): try: loop = asyncio.get_event_loop() except RuntimeError: return None - return id(asyncio.Task.current_task(loop=loop)) + return id(current_task(loop=loop)) @contextmanager From 42ee3e16a9ba7cc513b8da944cc1609a5407cf42 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 Nov 2019 20:19:01 -0800 Subject: [PATCH 63/81] Bump pint to 0.9 (#624) This fixes 2 deprecation warnings in Python 3.8 - refs #623 #622 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ae56306..e8229de1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( "click-default-group~=1.2.1", "Jinja2~=2.10.1", "hupper~=1.0", - "pint~=0.8.1", + "pint~=0.9", "pluggy~=0.12.0", "uvicorn~=0.8.4", "aiofiles~=0.4.0", From 5bc2570121aea8141ff88790e214765472882b08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 20:45:12 -0800 Subject: [PATCH 64/81] Include uvicorn version in /-/versions, refs #622 --- datasette/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 203e0991..4ba4adfb 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -12,6 +12,7 @@ from pathlib import Path import click from markupsafe import Markup from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader +import uvicorn from .views.base import DatasetteError, ureg, AsgiRouter from .views.database import DatabaseDownload, DatabaseView @@ -433,6 +434,7 @@ class Datasette: }, "datasette": datasette_version, "asgi": "3.0", + "uvicorn": uvicorn.__version__, "sqlite": { "version": sqlite_version, "fts_versions": fts_versions, From cf7776d36fbacefa874cbd6e5fcdc9fff7661203 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:09:11 -0800 Subject: [PATCH 65/81] Support Python 3.8, stop supporting Python 3.5 (#627) * Upgrade to uvicorn 0.10.4 * Drop support for Python 3.5 * Bump all dependencies to latest releases * Update docs to reflect we no longer support 3.5 * Removed code that skipped black unit test on 3.5 Closes #622 --- .travis.yml | 1 - README.md | 2 +- docs/contributing.rst | 2 +- docs/installation.rst | 7 +++++-- setup.py | 20 ++++++++++---------- tests/test_black.py | 7 +------ 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6b15b7e..0fc87d93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - "3.6" - "3.7" - "3.8" - - "3.5" # Executed for 3.5 AND 3.5 as the first "test" stage: script: diff --git a/README.md b/README.md index 9f85f1ba..14c9cfd6 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ sqlite-utils: a Python library and CLI tool for building SQLite databases](https pip3 install datasette -Datasette requires Python 3.5 or higher. We also have [detailed installation instructions](https://datasette.readthedocs.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://datasette.readthedocs.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/docs/contributing.rst b/docs/contributing.rst index 43834edc..078fd841 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -18,7 +18,7 @@ General guidelines Setting up a development environment ------------------------------------ -If you have Python 3.5 or higher installed on your computer (on OS X the easiest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. +If you have Python 3.6 or higher installed on your computer (on OS X the easiest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. If you want to use GitHub to publish your changes, first `create a fork of datasette `__ under your own GitHub account. diff --git a/docs/installation.rst b/docs/installation.rst index e65d8ee3..9ee7eb4e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -69,16 +69,19 @@ You can now run the new custom image like so:: You can confirm that the plugins are installed by visiting http://127.0.0.1:8001/-/plugins - Install using pip ----------------- -To run Datasette without Docker you will need Python 3.5 or higher. +To run Datasette without Docker you will need Python 3.6 or higher. You can install Datasette and its dependencies using ``pip``:: pip install datasette +The last version to support Python 3.5 was 0.30.2 - you can install that version like so:: + + pip install datasette==0.30.2 + If you want to install Datasette in its own virtual environment, use this:: python -mvenv datasette-venv diff --git a/setup.py b/setup.py index e8229de1..7a4cdcb3 100644 --- a/setup.py +++ b/setup.py @@ -42,12 +42,12 @@ setup( include_package_data=True, install_requires=[ "click~=7.0", - "click-default-group~=1.2.1", - "Jinja2~=2.10.1", - "hupper~=1.0", + "click-default-group~=1.2.2", + "Jinja2~=2.10.3", + "hupper~=1.9", "pint~=0.9", - "pluggy~=0.12.0", - "uvicorn~=0.8.4", + "pluggy~=0.13.0", + "uvicorn~=0.10.4", "aiofiles~=0.4.0", ], entry_points=""" @@ -58,11 +58,11 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest~=5.0.0", + "pytest~=5.2.2", "pytest-asyncio~=0.10.0", - "aiohttp~=3.5.3", - "beautifulsoup4~=4.6.1", - "asgiref~=3.1.2", + "aiohttp~=3.6.2", + "beautifulsoup4~=4.8.1", + "asgiref~=3.2.3", ] + maybe_black, }, @@ -74,8 +74,8 @@ setup( "Intended Audience :: End Users/Desktop", "Topic :: Database", "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.5", ], ) diff --git a/tests/test_black.py b/tests/test_black.py index 68e2dcc0..b5bfcfd0 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1,3 +1,4 @@ +import black from click.testing import CliRunner from pathlib import Path import pytest @@ -6,13 +7,7 @@ import sys code_root = Path(__file__).parent.parent -@pytest.mark.skipif( - sys.version_info[:2] < (3, 6), reason="Black requires Python 3.6 or later" -) def test_black(): - # Do not import at top of module because Python 3.5 will not have it installed - import black - runner = CliRunner() result = runner.invoke( black.main, [str(code_root / "tests"), str(code_root / "datasette"), "--check"] From 76fc6a9c7317ce4fbf3cc3d327c849f7274d960a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:17:59 -0800 Subject: [PATCH 66/81] Release notes for 0.31 --- docs/changelog.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4761efe..6e260be9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,28 @@ Changelog ========= +.. _v0_31: + +0.31 (2019-11-11) +----------------- + +This version adds compatibility with Python 3.8 and breaks compatibility with Python 3.5. + +If you are still running Python 3.5 you should stick with ``0.30.2``, which you can install like this:: + + pip install datasette==0.30.2 + +- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (`#602 `__) +- New ``?column__notin=x,y,z`` filter for table views (`#614 `__) +- Table view now uses ``select col1, col2, col3`` instead of ``select *`` +- Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 `__) +- Removed obsolete ``?_group_count=col`` feature (`#504 `__) +- Improved user interface and documentation for ``datasette publish cloudrun`` (`#608 `__) +- Tables with indexes now show the `` CREATE INDEX`` statements on the table page (`#618 `__) +- Current version of `uvicorn `__ is now shown on ``/-/versions`` +- Python 3.8 is now supported! (`#622 `__) +- Python 3.5 is no longer supported. + .. _v0_30_2: 0.30.2 (2019-11-02) From c633c035dc8d4c60f1d13cb074918406bbdb3734 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:26:56 -0800 Subject: [PATCH 67/81] Datasette 0.31 in news section --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 14c9cfd6..05995a74 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 11th November 2019: [Datasette 0.31](https://datasette.readthedocs.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... From 7f89928062b1a1fdb2625a946f7cd5161e597401 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:33:51 -0800 Subject: [PATCH 68/81] Removed code that conditionally installs black Since we no longer support Python 3.5 we don't need this any more. --- setup.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 7a4cdcb3..15284779 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,6 @@ def get_version(): return g["__version__"] -# Only install black on Python 3.6 or higher -maybe_black = [] -if sys.version_info > (3, 6): - maybe_black = ["black~=19.10b0"] - setup( name="datasette", version=versioneer.get_version(), @@ -63,8 +58,8 @@ setup( "aiohttp~=3.6.2", "beautifulsoup4~=4.8.1", "asgiref~=3.2.3", - ] - + maybe_black, + "black~=19.10b0", + ], }, tests_require=["datasette[test]"], classifiers=[ From 1c518680e9692a9a77022af54f3de3e77fb1aaf4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 21:57:48 -0800 Subject: [PATCH 69/81] Final steps: build stable branch of Read The Docs --- docs/contributing.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 078fd841..48930332 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -150,4 +150,7 @@ Wait long enough for Travis to build and deploy the demo version of that commit git tag 0.25.2 git push --tags -Once the release is out, you can manually update https://github.com/simonw/datasette/releases +Final steps once the release has deployed to https://pypi.org/project/datasette/ + +* Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases +* Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/ From f554be39fc14ddc18921ca29d3920d55aad03d46 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 22:00:13 -0800 Subject: [PATCH 70/81] ReST fix --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e260be9..763b178e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,7 +21,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you - Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 `__) - Removed obsolete ``?_group_count=col`` feature (`#504 `__) - Improved user interface and documentation for ``datasette publish cloudrun`` (`#608 `__) -- Tables with indexes now show the `` CREATE INDEX`` statements on the table page (`#618 `__) +- Tables with indexes now show the ``CREATE INDEX`` statements on the table page (`#618 `__) - Current version of `uvicorn `__ is now shown on ``/-/versions`` - Python 3.8 is now supported! (`#622 `__) - Python 3.5 is no longer supported. From d977fbadf70a96bf2eea1407d01f99d98e092dec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 Nov 2019 22:03:09 -0800 Subject: [PATCH 71/81] datasette publish uses python:3.8 base Docker image, closes #629 --- datasette/utils/__init__.py | 2 +- tests/test_publish_cloudrun.py | 2 +- tests/test_publish_now.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 3d28a36b..b8df48cf 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -306,7 +306,7 @@ def make_dockerfile( install = ["datasette"] + list(install) return """ -FROM python:3.6 +FROM python:3.8 COPY . /app WORKDIR /app {spatialite_extras} diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index a038b60e..c5b18cdf 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -128,7 +128,7 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): .split("\n====================\n")[0] .strip() ) - expected = """FROM python:3.6 + expected = """FROM python:3.8 COPY . /app WORKDIR /app diff --git a/tests/test_publish_now.py b/tests/test_publish_now.py index 72aa71db..27fd1245 100644 --- a/tests/test_publish_now.py +++ b/tests/test_publish_now.py @@ -138,7 +138,7 @@ def test_publish_now_plugin_secrets(mock_run, mock_which): .split("\n====================\n")[0] .strip() ) - expected = """FROM python:3.6 + expected = """FROM python:3.8 COPY . /app WORKDIR /app From 16265f6a1a7c547e3925e0fc2d6b88754afb0435 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:18:04 -0800 Subject: [PATCH 72/81] Release notes for 0.31.1 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 763b178e..746f5b42 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_31_1: + +0.31.1 (2019-11-12) +------------------- + +- Deploymens created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) + .. _v0_31: 0.31 (2019-11-11) From a22c7761b61baa61b8e3da7d30887468d61d6b83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:18:39 -0800 Subject: [PATCH 73/81] Fixed typo in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 746f5b42..e527518e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 0.31.1 (2019-11-12) ------------------- -- Deploymens created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) +- Deployments created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 `__) .. _v0_31: From bbd00e903cdd49067ecdbdb60a4d225833a44b05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Nov 2019 18:38:13 -0800 Subject: [PATCH 74/81] Badge linking to datasette on hub.docker.com --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 05995a74..9a22c2b2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/master/LICENSE) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://black.readthedocs.io/en/stable/) +[![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) *A tool for exploring and publishing data* From 848dec4deb0d3c140a4e0394cac45fbb2593349b Mon Sep 17 00:00:00 2001 From: Stanley Zheng Date: Tue, 12 Nov 2019 23:28:42 -0500 Subject: [PATCH 75/81] Fix for datasette publish with just --source_url (#631) Closes #572 --- datasette/templates/_description_source_license.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index 3327706e..a2bc18f2 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -21,7 +21,7 @@ {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %} {% endif %} - {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadat.source_url %}·{% endif %} + {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadata.source_url %}·{% endif %} About: {% if metadata.about_url %} {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %} From f52451023025579ae9a13de4a7f00d69200184cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Nov 2019 08:42:47 -0800 Subject: [PATCH 76/81] Fix "publish heroku" + upgrade to use Python 3.8.0 Closes #633. Closes #632. --- datasette/publish/heroku.py | 7 +++++-- tests/test_publish_heroku.py | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 34d1f773..e75f76df 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -72,7 +72,10 @@ def publish_subcommand(publish): "about_url": about_url, } - environment_variables = {} + environment_variables = { + # Avoid uvicorn error: https://github.com/simonw/datasette/issues/633 + "WEB_CONCURRENCY": "1" + } if plugin_secret: extra_metadata["plugins"] = {} for plugin_name, plugin_setting, setting_value in plugin_secret: @@ -164,7 +167,7 @@ def temporary_heroku_directory( if metadata_content: open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.6.8") + open("runtime.txt", "w").write("python-3.8.0") if branch: install = [ diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 4cd66219..87386e93 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -57,8 +57,13 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): open("test.db", "w").write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) assert 0 == result.exit_code, result.output - mock_call.assert_called_once_with( - ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + mock_call.assert_has_calls( + [ + mock.call(["heroku", "config:set", "-a", "f", "WEB_CONCURRENCY=1",]), + mock.call( + ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + ), + ] ) From b51f258d00bb3c3b401f15d46a1fbd50394dbe1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Nov 2019 08:48:36 -0800 Subject: [PATCH 77/81] Release notes for 0.31.2 --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e527518e..f4958399 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_31_2: + +0.31.2 (2019-11-13) +------------------- + +- Fixed a bug where ``datasette publish heroku`` applications failed to start (`#633 `__) +- Fix for ``datasette publish`` with just ``--source_url`` - thanks, Stanley Zheng (`#572 `__) +- Deployments to Heroku now use Python 3.8.0 (`#632 `__) + .. _v0_31_1: 0.31.1 (2019-11-12) From 8c642f04e0608bf537fdd1f76d64c2367fb04d57 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:14:22 -0800 Subject: [PATCH 78/81] Render templates using Jinja async mode Closes #628 --- datasette/app.py | 6 ++++-- datasette/views/base.py | 2 +- docs/plugins.rst | 23 ++++++++++++----------- tests/fixtures.py | 8 +++++++- tests/test_plugins.py | 18 ++++++++++++++++++ tests/test_templates/show_json.html | 1 + 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4ba4adfb..02fcf303 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -583,7 +583,9 @@ class Datasette: ), ] ) - self.jinja_env = Environment(loader=template_loader, autoescape=True) + self.jinja_env = Environment( + loader=template_loader, autoescape=True, enable_async=True + ) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite @@ -730,5 +732,5 @@ class DatasetteRouter(AsgiRouter): else: template = self.ds.jinja_env.select_template(templates) await asgi_send_html( - send, template.render(info), status=status, headers=headers + send, await template.render_async(info), status=status, headers=headers ) diff --git a/datasette/views/base.py b/datasette/views/base.py index 062c6956..5182479c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -139,7 +139,7 @@ class BaseView(AsgiView): extra_template_vars.update(extra_vars) return Response.html( - template.render( + await template.render_async( { **context, **{ diff --git a/docs/plugins.rst b/docs/plugins.rst index 6df7ff6a..e5a3d7dd 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -629,7 +629,9 @@ Function that returns a dictionary If you return a function it will be executed. If it returns a dictionary those values will will be merged into the template context. Function that returns an awaitable function that returns a dictionary - You can also return a function which returns an awaitable function which returns a dictionary. This means you can execute additional SQL queries using ``datasette.execute()``. + You can also return a function which returns an awaitable function which returns a dictionary. + +Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. Here's an example plugin that returns an authentication object from the ASGI scope: @@ -641,20 +643,19 @@ Here's an example plugin that returns an authentication object from the ASGI sco "auth": request.scope.get("auth") } -And here's an example which returns the current version of SQLite: +And here's an example which adds a ``sql_first(sql_query)`` function which executes a SQL statement and returns the first column of the first row of results: .. code-block:: python @hookimpl - def extra_template_vars(datasette): - async def inner(): - first_db = list(datasette.databases.keys())[0] - return { - "sqlite_version": ( - await datasette.execute(first_db, "select sqlite_version()") - ).rows[0][0] - } - return inner + def extra_template_vars(datasette, database): + async def sql_first(sql, dbname=None): + dbname = dbname or database or next(iter(datasette.databases.keys())) + return (await datasette.execute(dbname, sql)).rows[0][0] + +You can then use the new function in a template like so:: + + SQLite version: {{ sql_first("select sqlite_version()") }} .. _plugin_register_output_renderer: diff --git a/tests/fixtures.py b/tests/fixtures.py index 87e66f99..3e4203f7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -446,13 +446,19 @@ def render_cell(value, database): @hookimpl def extra_template_vars(template, database, table, view_name, request, datasette): + async def query_database(sql): + first_db = list(datasette.databases.keys())[0] + return ( + await datasette.execute(first_db, sql) + ).rows[0][0] async def inner(): return { "extra_template_vars_from_awaitable": json.dumps({ "template": template, "scope_path": request.scope["path"], "awaitable": True, - }, default=lambda b: b.decode("utf8")) + }, default=lambda b: b.decode("utf8")), + "query_database": query_database, } return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b1c7fd9a..42d063f4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import app_client, make_app_client, TEMP_PLUGIN_SECRET_FILE # noqa +from datasette.utils import sqlite3 import base64 import json import os @@ -214,3 +215,20 @@ def test_plugins_extra_template_vars(restore_working_directory): "awaitable": True, "scope_path": "/-/metadata", } == extra_template_vars_from_awaitable + + +def test_plugins_async_template_function(restore_working_directory): + for client in make_app_client( + template_dir=str(pathlib.Path(__file__).parent / "test_templates") + ): + response = client.get("/-/metadata") + assert response.status == 200 + extra_from_awaitable_function = ( + Soup(response.body, "html.parser") + .select("pre.extra_from_awaitable_function")[0] + .text + ) + expected = ( + sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] + ) + assert expected == extra_from_awaitable_function diff --git a/tests/test_templates/show_json.html b/tests/test_templates/show_json.html index bbf1bc06..cff04fb4 100644 --- a/tests/test_templates/show_json.html +++ b/tests/test_templates/show_json.html @@ -5,4 +5,5 @@ Test data for extra_template_vars: 
{{ extra_template_vars|safe }}
{{ extra_template_vars_from_awaitable|safe }}
+
{{ query_database("select sqlite_version();") }}
{% endblock %} From a95bedb9c423fa6d772c93ef47bc40f13a5bea50 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:18:53 -0800 Subject: [PATCH 79/81] Release notes for 0.32 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4958399..2f909364 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_32: + +0.32 (2019-11-14) +----------------- + +Datasette now renders templates using `Jinja async mode `__. This makes it easy for plugins to provide custom template functions that perform asynchronous actions, for example the new `datasette-template-sql `__ plugin which allows custom templates to directly execute SQL queries and render their results. (`#628 `__) + .. _v0_31_2: 0.31.2 (2019-11-13) From 8fc9a5d877d26dbf2654e125f407ddd2fd767335 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Nov 2019 15:46:37 -0800 Subject: [PATCH 80/81] Datasette 0.32 and datasette-template-sql in news --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a22c2b2..030c507f 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 14th November 2019: [Datasette 0.32](https://datasette.readthedocs.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function. * 11th November 2019: [Datasette 0.31](https://datasette.readthedocs.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30) * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. From a9909c29ccac771c23c2ef22b89d10697b5256b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Nov 2019 14:49:45 -0800 Subject: [PATCH 81/81] Move .execute() from Datasette to Database Refs #569 - I split this change out from #579 --- datasette/app.py | 90 ++++++--------------------- datasette/database.py | 137 +++++++++++++++++++++++++++++++----------- 2 files changed, 121 insertions(+), 106 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 02fcf303..119d0e19 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -24,13 +24,11 @@ from .database import Database from .utils import ( QueryInterrupted, - Results, escape_css_string, escape_sqlite, get_plugins, module_from_path, sqlite3, - sqlite_timelimit, to_css_class, ) from .utils.asgi import ( @@ -42,13 +40,12 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) -from .tracer import trace, AsgiTracer +from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS from .version import __version__ app_root = Path(__file__).parent.parent -connections = threading.local() MEMORY = object() ConfigOption = collections.namedtuple("ConfigOption", ("name", "default", "help")) @@ -336,6 +333,25 @@ class Datasette: # pylint: disable=no-member pm.hook.prepare_connection(conn=conn) + async def execute( + self, + db_name, + sql, + params=None, + truncate=False, + custom_time_limit=None, + page_size=None, + log_sql_errors=True, + ): + return await self.databases[db_name].execute( + sql, + params=params, + truncate=truncate, + custom_time_limit=custom_time_limit, + page_size=page_size, + log_sql_errors=log_sql_errors, + ) + async def expand_foreign_keys(self, database, table, column, values): "Returns dict mapping (column, value) -> label" labeled_fks = {} @@ -477,72 +493,6 @@ class Datasette: .get(table, {}) ) - async def execute_against_connection_in_thread(self, db_name, fn): - def in_thread(): - conn = getattr(connections, db_name, None) - if not conn: - conn = self.databases[db_name].connect() - self.prepare_connection(conn) - setattr(connections, db_name, conn) - return fn(conn) - - return await asyncio.get_event_loop().run_in_executor(self.executor, in_thread) - - async def execute( - self, - db_name, - sql, - params=None, - truncate=False, - custom_time_limit=None, - page_size=None, - log_sql_errors=True, - ): - """Executes sql against db_name in a thread""" - page_size = page_size or self.page_size - - def sql_operation_in_thread(conn): - time_limit_ms = self.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: - time_limit_ms = custom_time_limit - - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params or {}) - max_returned_rows = self.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except sqlite3.OperationalError as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - print( - "ERROR: conn={}, sql = {}, params = {}: {}".format( - conn, repr(sql), params, e - ) - ) - raise - - if truncate: - return Results(rows, truncated, cursor.description) - - else: - return Results(rows, False, cursor.description) - - with trace("sql", database=db_name, sql=sql.strip(), params=params): - results = await self.execute_against_connection_in_thread( - db_name, sql_operation_in_thread - ) - return results - def register_renderers(self): """ Register output renderers which output data in custom formats. """ # Built-in renderers diff --git a/datasette/database.py b/datasette/database.py index 3a1cea94..9a8ae4d4 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,17 +1,25 @@ +import asyncio +import contextlib from pathlib import Path +import threading +from .tracer import trace from .utils import ( QueryInterrupted, + Results, detect_fts, detect_primary_keys, detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, + sqlite_timelimit, sqlite3, table_columns, ) from .inspect import inspect_hash +connections = threading.local() + class Database: def __init__(self, ds, path=None, is_mutable=False, is_memory=False): @@ -45,6 +53,73 @@ class Database: "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False ) + async def execute_against_connection_in_thread(self, fn): + def in_thread(): + conn = getattr(connections, self.name, None) + if not conn: + conn = self.connect() + self.ds.prepare_connection(conn) + setattr(connections, self.name, conn) + return fn(conn) + + return await asyncio.get_event_loop().run_in_executor( + self.ds.executor, in_thread + ) + + async def execute( + self, + sql, + params=None, + truncate=False, + custom_time_limit=None, + page_size=None, + log_sql_errors=True, + ): + """Executes sql against db_name in a thread""" + page_size = page_size or self.ds.page_size + + def sql_operation_in_thread(conn): + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < time_limit_ms: + time_limit_ms = custom_time_limit + + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params or {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except sqlite3.OperationalError as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + print( + "ERROR: conn={}, sql = {}, params = {}: {}".format( + conn, repr(sql), params, e + ) + ) + raise + + if truncate: + return Results(rows, truncated, cursor.description) + + else: + return Results(rows, False, cursor.description) + + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_against_connection_in_thread( + sql_operation_in_thread + ) + return results + @property def size(self): if self.is_memory: @@ -62,8 +137,7 @@ class Database: for table in await self.table_names(): try: table_count = ( - await self.ds.execute( - self.name, + await self.execute( "select count(*) from [{}]".format(table), custom_time_limit=limit, ) @@ -89,32 +163,30 @@ class Database: return Path(self.path).stem async def table_exists(self, table): - results = await self.ds.execute( - self.name, - "select 1 from sqlite_master where type='table' and name=?", - params=(table,), + results = await self.execute( + "select 1 from sqlite_master where type='table' and name=?", params=(table,) ) return bool(results.rows) async def table_names(self): - results = await self.ds.execute( - self.name, "select name from sqlite_master where type='table'" + results = await self.execute( + "select name from sqlite_master where type='table'" ) return [r[0] for r in results.rows] async def table_columns(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: table_columns(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: table_columns(conn, table) ) async def primary_keys(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: detect_primary_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: detect_primary_keys(conn, table) ) async def fts_table(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: detect_fts(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: detect_fts(conn, table) ) async def label_column_for_table(self, table): @@ -124,8 +196,8 @@ class Database: if explicit_label_column: return explicit_label_column # If a table has two columns, one of which is ID, then label_column is the other one - column_names = await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: table_columns(conn, table) + column_names = await self.execute_against_connection_in_thread( + lambda conn: table_columns(conn, table) ) # Is there a name or title column? name_or_title = [c for c in column_names if c in ("name", "title")] @@ -141,8 +213,8 @@ class Database: return None async def foreign_keys_for_table(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: get_outbound_foreign_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: get_outbound_foreign_keys(conn, table) ) async def hidden_table_names(self): @@ -150,18 +222,17 @@ class Database: hidden_tables = [ r[0] for r in ( - await self.ds.execute( - self.name, + await self.execute( """ select name from sqlite_master where rootpage = 0 and sql like '%VIRTUAL TABLE%USING FTS%' - """, + """ ) ).rows ] - has_spatialite = await self.ds.execute_against_connection_in_thread( - self.name, detect_spatialite + has_spatialite = await self.execute_against_connection_in_thread( + detect_spatialite ) if has_spatialite: # Also hide Spatialite internal tables @@ -178,13 +249,12 @@ class Database: ] + [ r[0] for r in ( - await self.ds.execute( - self.name, + await self.execute( """ select name from sqlite_master where name like "idx_%" and type = "table" - """, + """ ) ).rows ] @@ -207,25 +277,20 @@ class Database: return hidden_tables async def view_names(self): - results = await self.ds.execute( - self.name, "select name from sqlite_master where type='view'" - ) + results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] async def get_all_foreign_keys(self): - return await self.ds.execute_against_connection_in_thread( - self.name, get_all_foreign_keys - ) + return await self.execute_against_connection_in_thread(get_all_foreign_keys) async def get_outbound_foreign_keys(self, table): - return await self.ds.execute_against_connection_in_thread( - self.name, lambda conn: get_outbound_foreign_keys(conn, table) + return await self.execute_against_connection_in_thread( + lambda conn: get_outbound_foreign_keys(conn, table) ) async def get_table_definition(self, table, type_="table"): table_definition_rows = list( - await self.ds.execute( - self.name, + await self.execute( "select sql from sqlite_master where name = :n and type=:t", {"n": table, "t": type_}, )