Merge "Support filtering with units" from #205

datasette/app.py

@@ -85,6 +85,12 @@ class BaseView(RenderMixin):
         self.page_size = datasette.page_size
         self.max_returned_rows = datasette.max_returned_rows
 
+    def table_metadata(self, database, table):
+        "Fetch table-specific metadata."
+        return self.ds.metadata.get(
+            'databases', {}
+        ).get(database, {}).get('tables', {}).get(table, {})
+
     def options(self, request, *args, **kwargs):
         r = response.text('ok')
         if self.ds.cors:
@@ -450,11 +456,6 @@ class DatabaseDownload(BaseView):
 
 
 class RowTableShared(BaseView):
-    def table_metadata(self, database, table):
-        return self.ds.metadata.get(
-            'databases', {}
-        ).get(database, {}).get('tables', {}).get(table, {})
-
     def sortable_columns_for_table(self, name, table, use_rowid):
         table_metadata = self.table_metadata(name, table)
         if 'sortable_columns' in table_metadata:
@@ -657,7 +658,9 @@ class TableView(RowTableShared):
                 forward_querystring=False
             )
 
-        filters = Filters(sorted(other_args.items()))
+        units = self.table_metadata(name, table).get('units', {})
+
+        filters = Filters(sorted(other_args.items()), units, ureg)
         where_clauses, params = filters.build_where_clauses()
 
         # _search support:
@@ -901,6 +904,7 @@ class TableView(RowTableShared):
             'filtered_table_rows_count': filtered_table_rows_count,
             'columns': columns,
             'primary_keys': pks,
+            'units': units,
             'query': {
                 'sql': sql,
                 'params': params,
@@ -970,6 +974,7 @@ class RowView(RowTableShared):
             'columns': columns,
             'primary_keys': pks,
             'primary_key_values': pk_values,
+            'units': self.table_metadata(name, table).get('units', {})
         }
 
         if 'foreign_key_tables' in (request.raw_args.get('_extras') or '').split(','):
@@ -1195,6 +1200,11 @@ class Datasette:
                 }
         return self._inspect
 
+    def register_custom_units(self):
+        "Register any custom units defined in the metadata.json with Pint"
+        for unit in self.metadata.get('custom_units', []):
+            ureg.define(unit)
+
     def app(self):
         app = Sanic(__name__)
         default_templates = str(app_root / 'datasette' / 'templates')
@@ -1239,6 +1249,8 @@ class Datasette:
             '/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_json:(\.jsono?)?$>'
         )
 
+        self.register_custom_units()
+
         @app.exception(Exception)
         def on_exception(request, exception):
             title = None

datasette/cli.py

@@ -160,6 +160,7 @@ def skeleton(files, metadata, sqlite_extensions):
                     'license_url': None,
                     'source': None,
                     'source_url': None,
+                    'units': {}
                 } for table_name in (info.get('tables') or {})
             }
         }

datasette/utils.py

@@ -10,6 +10,7 @@ import tempfile
 import time
 import shutil
 import urllib
+import numbers
 
 
 # From https://www.sqlite.org/lang_keywords.html
@@ -459,8 +460,10 @@ class Filters:
         f.key: f for f in _filters
     }
 
-    def __init__(self, pairs):
+    def __init__(self, pairs, units={}, ureg=None):
         self.pairs = pairs
+        self.units = units
+        self.ureg = ureg
 
     def lookups(self):
         "Yields (lookup, display, no_argument) pairs"
@@ -500,13 +503,27 @@ class Filters:
     def has_selections(self):
         return bool(self.pairs)
 
+    def convert_unit(self, column, value):
+        "If the user has provided a unit in the quey, convert it into the column unit, if present."
+        if column not in self.units:
+            return value
+
+        # Try to interpret the value as a unit
+        value = self.ureg(value)
+        if isinstance(value, numbers.Number):
+            # It's just a bare number, assume it's the column unit
+            return value
+
+        column_unit = self.ureg(self.units[column])
+        return value.to(column_unit).magnitude
+
     def build_where_clauses(self):
         sql_bits = []
         params = {}
         for i, (column, lookup, value) in enumerate(self.selections()):
             filter = self._filters_by_key.get(lookup, None)
             if filter:
-                sql_bit, param = filter.where_clause(column, value, i)
+                sql_bit, param = filter.where_clause(column, self.convert_unit(column, value), i)
                 sql_bits.append(sql_bit)
                 if param is not None:
                     param_id = 'p{}'.format(i)

docs/metadata.rst

@@ -55,6 +55,43 @@ You can also provide metadata at the per-database or per-table level, like this:
 
 Each of the top-level metadata fields can be used at the database and table level.
 
+Specifying units for a column
+-----------------------------
+
+Datasette supports attaching units to a column, which will be used when displaying
+values from that column. SI prefixes will be used where appropriate.
+
+Column units are configured in the metadata like so::
+
+    {
+        "databases": {
+            "database1": {
+                "tables": {
+                    "example_table": {
+                        "units": {
+                            "column1": "metres",
+                            "column2": "Hz"
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+Units are interpreted using Pint_, and you can see the full list of available units in
+Pint's `unit registry`_. You can also add `custom units`_ to the metadata, which will be
+registered with Pint::
+    
+    {
+        "custom_units": [
+            "decibel = [] = dB"
+        ]
+    }
+
+.. _Pint: https://pint.readthedocs.io/
+.. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt
+.. _custom units: http://pint.readthedocs.io/en/latest/defining.html
+
 Setting which columns can be used for sorting
 ---------------------------------------------
 
@@ -119,7 +156,8 @@ This will create a ``metadata.json`` file looking something like this::
                         "license": null,
                         "license_url": null,
                         "source": null,
-                        "source_url": null
+                        "source_url": null,
+                        "units": {}
                     }
                 }
             },

tests/fixtures.py

@@ -79,6 +79,12 @@ METADATA = {
                 'no_primary_key': {
                     'sortable_columns': [],
                 },
+                'units': {
+                    'units': {
+                        'distance': 'm',
+                        'frequency': 'Hz'
+                    }
+                },
             }
         },
     }
@@ -169,6 +175,16 @@ CREATE TABLE "complex_foreign_keys" (
   FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
 );
 
+CREATE TABLE units (
+  pk integer primary key,
+  distance int,
+  frequency int
+);
+
+INSERT INTO units VALUES (1, 1, 100);
+INSERT INTO units VALUES (2, 5000, 2500);
+INSERT INTO units VALUES (3, 100000, 75000);
+
 CREATE TABLE [select] (
   [group] text,
   [having] text,
@@ -207,7 +223,6 @@ CREATE VIEW simple_view AS
     ).replace('None', 'null') for row in generate_sortable_rows(201)
 ])
 
-
 if __name__ == '__main__':
     filename = sys.argv[-1]
     if filename.endswith('.db'):

tests/test_api.py

@@ -14,7 +14,7 @@ def test_homepage(app_client):
     assert response.json.keys() == {'test_tables': 0}.keys()
     d = response.json['test_tables']
     assert d['name'] == 'test_tables'
-    assert d['tables_count'] == 12
+    assert d['tables_count'] == 13
 
 
 def test_database_page(app_client):
@@ -172,6 +172,14 @@ def test_database_page(app_client):
         'foreign_keys': {'incoming': [], 'outgoing': []},
         'label_column': None,
         'primary_keys': ['pk'],
+    }, {
+        'columns': ['pk', 'distance', 'frequency'],
+        'name': 'units',
+        'count': 3,
+        'hidden': False,
+        'foreign_keys': {'incoming': [], 'outgoing': []},
+        'label_column': None,
+        'primary_keys': ['pk'],
     }] == data['tables']
 
 
@@ -577,3 +585,16 @@ def test_row_foreign_key_tables(app_client):
         'other_column': 'f1',
         'other_table': 'complex_foreign_keys'
     }] == response.json['foreign_key_tables']
+
+
+def test_unit_filters(app_client):
+    response = app_client.get('/test_tables/units.json?distance__lt=75km&frequency__gt=1kHz',
+                              gather_request=False)
+    assert response.status == 200
+    data = response.json
+
+    assert data['units']['distance'] == 'm'
+    assert data['units']['frequency'] == 'Hz'
+
+    assert len(data['rows']) == 1
+    assert data['rows'][0][0] == 2