From 8bfeb984788c7144088c16c5f9126ca7d6af6e93 Mon Sep 17 00:00:00 2001 From: Russ Garrett Date: Sat, 14 Apr 2018 11:16:09 +0100 Subject: [PATCH 1/5] Tidy up units support * Add units to exported JSON * Units key in metadata skeleton * Docs --- datasette/app.py | 13 ++++++++----- datasette/cli.py | 1 + docs/metadata.rst | 32 +++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 6b143785..0560193e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -84,6 +84,12 @@ class BaseView(RenderMixin): self.page_size = datasette.page_size self.max_returned_rows = datasette.max_returned_rows + def table_metadata(self, database, table): + "Fetch table-specific metadata." + return self.ds.metadata.get( + 'databases', {} + ).get(database, {}).get('tables', {}).get(table, {}) + def options(self, request, *args, **kwargs): r = response.text('ok') if self.ds.cors: @@ -449,11 +455,6 @@ class DatabaseDownload(BaseView): class RowTableShared(BaseView): - def table_metadata(self, database, table): - return self.ds.metadata.get( - 'databases', {} - ).get(database, {}).get('tables', {}).get(table, {}) - def sortable_columns_for_table(self, name, table, use_rowid): table_metadata = self.table_metadata(name, table) if 'sortable_columns' in table_metadata: @@ -890,6 +891,7 @@ class TableView(RowTableShared): 'filtered_table_rows_count': filtered_table_rows_count, 'columns': columns, 'primary_keys': pks, + 'units': self.table_metadata(name, table).get('units', {}), 'query': { 'sql': sql, 'params': params, @@ -959,6 +961,7 @@ class RowView(RowTableShared): 'columns': columns, 'primary_keys': pks, 'primary_key_values': pk_values, + 'units': self.table_metadata(name, table).get('units', {}) } if 'foreign_key_tables' in (request.raw_args.get('_extras') or '').split(','): diff --git a/datasette/cli.py b/datasette/cli.py index 5744d00a..0fde53cf 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -160,6 +160,7 @@ def skeleton(files, metadata, sqlite_extensions): 'license_url': None, 'source': None, 'source_url': None, + 'units': {} } for table_name in (info.get('tables') or {}) } } diff --git a/docs/metadata.rst b/docs/metadata.rst index 3c842634..0f94311c 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -55,6 +55,35 @@ You can also provide metadata at the per-database or per-table level, like this: Each of the top-level metadata fields can be used at the database and table level. +Specifying units for a column +----------------------------- + +Datasette supports attaching units to a column, which will be used when displaying +values from that column. SI prefixes will be used where appropriate. + +Column units are configured in the metadata like so:: + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "units": { + "column1": "metres", + "column2": "Hz" + } + } + } + } + } + } + +Units are interpreted using Pint_, and you can see the full list of available units in +Pint's `unit registry`_. + +.. _Pint: https://pint.readthedocs.io/ +.. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt + Setting which columns can be used for sorting --------------------------------------------- @@ -119,7 +148,8 @@ This will create a ``metadata.json`` file looking something like this:: "license": null, "license_url": null, "source": null, - "source_url": null + "source_url": null, + "units": {} } } }, From ab85605c6179b21bb0add59b76c1b376d9d248b2 Mon Sep 17 00:00:00 2001 From: Russ Garrett Date: Sat, 14 Apr 2018 11:41:27 +0100 Subject: [PATCH 2/5] Support units in filters --- datasette/app.py | 6 ++++-- datasette/utils.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0560193e..bd398234 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -647,7 +647,9 @@ class TableView(RowTableShared): forward_querystring=False ) - filters = Filters(sorted(other_args.items())) + units = self.table_metadata(name, table).get('units', {}) + + filters = Filters(sorted(other_args.items()), units, ureg) where_clauses, params = filters.build_where_clauses() # _search support: @@ -891,7 +893,7 @@ class TableView(RowTableShared): 'filtered_table_rows_count': filtered_table_rows_count, 'columns': columns, 'primary_keys': pks, - 'units': self.table_metadata(name, table).get('units', {}), + 'units': units, 'query': { 'sql': sql, 'params': params, diff --git a/datasette/utils.py b/datasette/utils.py index cd1f08cf..b5020be2 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -10,6 +10,7 @@ import tempfile import time import shutil import urllib +import numbers # From https://www.sqlite.org/lang_keywords.html @@ -459,8 +460,10 @@ class Filters: f.key: f for f in _filters } - def __init__(self, pairs): + def __init__(self, pairs, units={}, ureg=None): self.pairs = pairs + self.units = units + self.ureg = ureg def lookups(self): "Yields (lookup, display, no_argument) pairs" @@ -500,13 +503,27 @@ class Filters: def has_selections(self): return bool(self.pairs) + def convert_unit(self, column, value): + "If the user has provided a unit in the quey, convert it into the column unit, if present." + if column not in self.units: + return value + + # Try to interpret the value as a unit + value = self.ureg(value) + if isinstance(value, numbers.Number): + # It's just a bare number, assume it's the column unit + return value + + column_unit = self.ureg(self.units[column]) + return value.to(column_unit).magnitude + def build_where_clauses(self): sql_bits = [] params = {} for i, (column, lookup, value) in enumerate(self.selections()): filter = self._filters_by_key.get(lookup, None) if filter: - sql_bit, param = filter.where_clause(column, value, i) + sql_bit, param = filter.where_clause(column, self.convert_unit(column, value), i) sql_bits.append(sql_bit) if param is not None: param_id = 'p{}'.format(i) From 3c985ec271cce46db8682c2a7466ea1c9a2ed210 Mon Sep 17 00:00:00 2001 From: Russ Garrett Date: Sat, 14 Apr 2018 12:27:06 +0100 Subject: [PATCH 3/5] Allow custom units to be registered with Pint --- datasette/app.py | 7 +++++++ docs/metadata.rst | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index bd398234..bdaf3d4b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1186,6 +1186,11 @@ class Datasette: } return self._inspect + def register_custom_units(self): + "Register any custom units defined in the metadata.json with Pint" + for unit in self.metadata.get('custom_units', []): + ureg.define(unit) + def app(self): app = Sanic(__name__) default_templates = str(app_root / 'datasette' / 'templates') @@ -1230,6 +1235,8 @@ class Datasette: '///' ) + self.register_custom_units() + @app.exception(Exception) def on_exception(request, exception): title = None diff --git a/docs/metadata.rst b/docs/metadata.rst index 0f94311c..837359df 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -79,7 +79,14 @@ Column units are configured in the metadata like so:: } Units are interpreted using Pint_, and you can see the full list of available units in -Pint's `unit registry`_. +Pint's `unit registry`_. You can also add custom units to the metadata, which will be +registered with Pint:: + + { + "custom_units": [ + "decibel = [] = dB" + ] + } .. _Pint: https://pint.readthedocs.io/ .. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt From ed974417ad54f0c0f65b2f1cf54dc12485abb570 Mon Sep 17 00:00:00 2001 From: Russ Garrett Date: Sat, 14 Apr 2018 15:06:52 +0100 Subject: [PATCH 4/5] Tests for unit filtering --- tests/fixtures.py | 17 ++++++++++++++++- tests/test_api.py | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index b44b2e1d..37015bf1 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -79,6 +79,12 @@ METADATA = { 'no_primary_key': { 'sortable_columns': [], }, + 'units': { + 'units': { + 'distance': 'm', + 'frequency': 'Hz' + } + }, } }, } @@ -155,6 +161,16 @@ CREATE TABLE "complex_foreign_keys" ( FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) ); +CREATE TABLE units ( + pk integer primary key, + distance int, + frequency int +); + +INSERT INTO units VALUES (1, 1, 100); +INSERT INTO units VALUES (2, 5000, 2500); +INSERT INTO units VALUES (3, 100000, 75000); + CREATE TABLE [select] ( [group] text, [having] text, @@ -189,7 +205,6 @@ CREATE VIEW simple_view AS ).replace('None', 'null') for row in generate_sortable_rows(201) ]) - if __name__ == '__main__': filename = sys.argv[-1] if filename.endswith('.db'): diff --git a/tests/test_api.py b/tests/test_api.py index 33938667..608c69d2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -14,7 +14,7 @@ def test_homepage(app_client): assert response.json.keys() == {'test_tables': 0}.keys() d = response.json['test_tables'] assert d['name'] == 'test_tables' - assert d['tables_count'] == 10 + assert d['tables_count'] == 11 def test_database_page(app_client): @@ -134,6 +134,14 @@ def test_database_page(app_client): 'foreign_keys': {'incoming': [], 'outgoing': []}, 'label_column': None, 'primary_keys': ['pk'], + }, { + 'columns': ['pk', 'distance', 'frequency'], + 'name': 'units', + 'count': 3, + 'hidden': False, + 'foreign_keys': {'incoming': [], 'outgoing': []}, + 'label_column': None, + 'primary_keys': ['pk'], }] == data['tables'] @@ -534,3 +542,16 @@ def test_row_foreign_key_tables(app_client): 'other_column': 'f1', 'other_table': 'complex_foreign_keys' }] == response.json['foreign_key_tables'] + + +def test_unit_filters(app_client): + response = app_client.get('/test_tables/units.json?distance__lt=75km&frequency__gt=1kHz', + gather_request=False) + assert response.status == 200 + data = response.json + + assert data['units']['distance'] == 'm' + assert data['units']['frequency'] == 'Hz' + + assert len(data['rows']) == 1 + assert data['rows'][0][0] == 2 From 7d5f25dfb320af26ec4afcb1a687f7f034e1f044 Mon Sep 17 00:00:00 2001 From: Russ Garrett Date: Sat, 14 Apr 2018 15:08:20 +0100 Subject: [PATCH 5/5] Add link to pint custom units page to docs --- docs/metadata.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/metadata.rst b/docs/metadata.rst index 837359df..7872f88f 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -79,7 +79,7 @@ Column units are configured in the metadata like so:: } Units are interpreted using Pint_, and you can see the full list of available units in -Pint's `unit registry`_. You can also add custom units to the metadata, which will be +Pint's `unit registry`_. You can also add `custom units`_ to the metadata, which will be registered with Pint:: { @@ -90,6 +90,7 @@ registered with Pint:: .. _Pint: https://pint.readthedocs.io/ .. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt +.. _custom units: http://pint.readthedocs.io/en/latest/defining.html Setting which columns can be used for sorting ---------------------------------------------