Show facets that timed out using new InterruptedError

If the user requests some _facet= options that do not successfully execute in
the configured facet_time_limit_ms, we now show a warning message like this:

    These facets timed out: rowid, Title

To build this I had to clean up our SQLite interrupted logic. We now raise a
custom InterruptedError exception when SQLite terminates due to exceeding a
time limit.

In implementing this I found and fixed a logic error where invalid SQL was
being generated in some cases for our faceting calculations but the resulting
sqlite3.OperationalError had been incorrectly captured and treated as a
timeout.

Refs #255
Closes #269
pull/258/merge
Simon Willison 2018-05-17 23:07:45 -07:00
rodzic cef9a9a870
commit 08f4b7658f
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 17E2DEA2588B7F52
4 zmienionych plików z 34 dodań i 16 usunięć

Wyświetl plik

@ -100,6 +100,10 @@
</p>
{% endif %}
{% if facets_timed_out %}
<p class="facets-timed-out">These facets timed out: {{ ", ".join(facets_timed_out) }}</p>
{% endif %}
{% if facet_results %}
<div class="facet-results">
{% for facet_info in sorted_facet_results %}

Wyświetl plik

@ -32,6 +32,10 @@ reserved_words = set((
).split())
class InterruptedError(Exception):
pass
def urlsafe_components(token):
"Splits token on commas and URL decodes each component"
return [

Wyświetl plik

@ -13,6 +13,7 @@ from sanic.views import HTTPMethodView
from datasette import __version__
from datasette.utils import (
CustomJSONEncoder,
InterruptedError,
InvalidSql,
path_from_row_pks,
path_with_added_args,
@ -170,7 +171,9 @@ class BaseView(RenderMixin):
else:
rows = cursor.fetchall()
truncated = False
except Exception as e:
except sqlite3.OperationalError as e:
if e.args == ('interrupted',):
raise InterruptedError(e)
print(
"ERROR: conn={}, sql = {}, params = {}: {}".format(
conn, repr(sql), params, e
@ -216,6 +219,8 @@ class BaseView(RenderMixin):
else:
data, extra_template_data, templates = response_or_template_contexts
except InterruptedError as e:
raise DatasetteError(str(e), title="SQL Interrupted", status=400)
except (sqlite3.OperationalError, InvalidSql) as e:
raise DatasetteError(str(e), title="Invalid SQL", status=400)

Wyświetl plik

@ -7,6 +7,7 @@ from sanic.request import RequestParameters
from datasette.utils import (
Filters,
InterruptedError,
compound_keys_after_sql,
escape_sqlite,
filters_should_redirect,
@ -16,7 +17,7 @@ from datasette.utils import (
path_with_removed_args,
path_with_replaced_args,
to_css_class,
urlsafe_components
urlsafe_components,
)
from .base import BaseView, DatasetteError, ureg
@ -75,8 +76,7 @@ class RowTableShared(BaseView):
results = await self.execute(
database, sql, list(set(values))
)
except sqlite3.OperationalError:
# Probably hit the timelimit
except InterruptedError:
pass
else:
for id, value in results:
@ -135,8 +135,7 @@ class RowTableShared(BaseView):
results = await self.execute(
database, sql, list(set(ids_to_lookup))
)
except sqlite3.OperationalError:
# Probably hit the timelimit
except InterruptedError:
pass
else:
for id, value in results:
@ -409,6 +408,10 @@ class TableView(RowTableShared):
"where {} ".format(" and ".join(where_clauses))
) if where_clauses else "",
)
# Store current params and where_clauses for later:
from_sql_params = dict(**params)
from_sql_where_clauses = where_clauses[:]
count_sql = "select count(*) {}".format(from_sql)
_next = special_args.get("_next")
@ -544,6 +547,7 @@ class TableView(RowTableShared):
except KeyError:
pass
facet_results = {}
facets_timed_out = []
for column in facets:
facet_sql = """
select {col} as value, count(*) as count
@ -552,7 +556,7 @@ class TableView(RowTableShared):
""".format(
col=escape_sqlite(column),
from_sql=from_sql,
and_or_where='and' if where_clauses else 'where',
and_or_where='and' if where_clause else 'where',
limit=facet_size+1,
)
try:
@ -595,9 +599,8 @@ class TableView(RowTableShared):
),
"selected": selected,
})
except sqlite3.OperationalError:
# Hit time limit
pass
except InterruptedError:
facets_timed_out.append(column)
columns = [r[0] for r in description]
rows = list(rows)
@ -638,10 +641,11 @@ class TableView(RowTableShared):
filtered_table_rows_count = None
if count_sql:
try:
count_rows = list(await self.execute(name, count_sql, params))
count_rows = list(await self.execute(
name, count_sql, from_sql_params
))
filtered_table_rows_count = count_rows[0][0]
except sqlite3.OperationalError:
# Almost certainly hit the timeout
except InterruptedError:
pass
# Detect suggested facets
@ -656,13 +660,13 @@ class TableView(RowTableShared):
'''.format(
column=escape_sqlite(facet_column),
from_sql=from_sql,
and_or_where='and' if where_clauses else 'where',
and_or_where='and' if from_sql_where_clauses else 'where',
limit=facet_size+1
)
distinct_values = None
try:
distinct_values = await self.execute(
name, suggested_facet_sql, params,
name, suggested_facet_sql, from_sql_params,
truncate=False,
custom_time_limit=self.ds.limits["facet_suggest_time_limit_ms"],
)
@ -679,7 +683,7 @@ class TableView(RowTableShared):
request, {'_facet': facet_column}
),
})
except sqlite3.OperationalError:
except InterruptedError:
pass
# human_description_en combines filters AND search, if provided
@ -717,6 +721,7 @@ class TableView(RowTableShared):
"display_columns": display_columns,
"filter_columns": filter_columns,
"display_rows": display_rows,
"facets_timed_out": facets_timed_out,
"sorted_facet_results": sorted(
facet_results.values(),
key=lambda f: (len(f["results"]), f["name"]),