From 0abd3abacb309a2bd5913a7a2df4e9256585b1bb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Apr 2018 07:52:54 -0700 Subject: [PATCH] New ?_shape=objects/object/lists param for JSON API (#192) New _shape= parameter replacing old .jsono extension Now instead of this: /database/table.jsono We use the _shape parameter like this: /database/table.json?_shape=objects Also introduced a new _shape called 'object' which looks like this: /database/table.json?_shape=object Returning an object for the rows key: ... "rows": { "pk1": { ... }, "pk2": { ... } } Refs #122 --- README.md | 2 +- datasette/app.py | 38 +++++++++++- datasette/templates/query.html | 2 +- datasette/templates/row.html | 2 +- datasette/templates/table.html | 2 +- datasette/utils.py | 5 +- docs/getting_started.rst | 2 +- docs/json_api.rst | 105 +++++++++++++++++++++++++++++++ tests/test_api.py | 109 ++++++++++++++++++++++++++++----- 9 files changed, 244 insertions(+), 23 deletions(-) create mode 100644 docs/json_api.rst diff --git a/README.md b/README.md index 06ddfaa2..568561a8 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ http://localhost:8001/History/downloads.json will return that data as JSON: } -http://localhost:8001/History/downloads.jsono will return that data as JSON in a more convenient but less efficient format: +http://localhost:8001/History/downloads.json?_shape=objects will return that data as JSON in a more convenient but less efficient format: { ... diff --git a/datasette/app.py b/datasette/app.py index 8e7cfa07..64aabe0c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -221,8 +221,20 @@ class BaseView(RenderMixin): if value: data[key] = value if as_json: - # Special case for .jsono extension + # Special case for .jsono extension - redirect to _shape=objects if as_json == '.jsono': + return self.redirect( + request, + path_with_added_args( + request, + {'_shape': 'objects'}, + path=request.path.rsplit('.jsono', 1)[0] + '.json' + ), + forward_querystring=False + ) + # Deal with the _shape option + shape = request.args.get('_shape', 'lists') + if shape in ('objects', 'object'): columns = data.get('columns') rows = data.get('rows') if rows and columns: @@ -230,6 +242,28 @@ class BaseView(RenderMixin): dict(zip(columns, row)) for row in rows ] + if shape == 'object': + error = None + if 'primary_keys' not in data: + error = '_shape=object is only available on tables' + else: + pks = data['primary_keys'] + if not pks: + error = '_shape=object not available for tables with no primary keys' + else: + object_rows = {} + for row in data['rows']: + pk_string = path_from_row_pks(row, pks, not pks) + object_rows[pk_string] = row + data['rows'] = object_rows + if error: + data = { + 'ok': False, + 'error': error, + 'database': name, + 'database_hash': hash, + } + headers = {} if self.ds.cors: headers['Access-Control-Allow-Origin'] = '*' @@ -278,6 +312,8 @@ class BaseView(RenderMixin): params = request.raw_args if 'sql' in params: params.pop('sql') + if '_shape' in params: + params.pop('_shape') # Extract any :named parameters named_parameters = self.re_named_parameter.findall(sql) named_parameter_values = { diff --git a/datasette/templates/query.html b/datasette/templates/query.html index fa850736..62bf41da 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -40,7 +40,7 @@ {% if rows %} -

This data as .json, .jsono

+

This data as .json

diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 53cbd15d..2217a4dc 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,7 +22,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -

This data as .json, .jsono

+

This data as .json

{% include custom_rows_and_columns_templates %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 70262bc7..546e852f 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -74,7 +74,7 @@

✎ View and edit SQL

{% endif %} -

This data as .json, .jsono

+

This data as .json

{% include custom_rows_and_columns_templates %} diff --git a/datasette/utils.py b/datasette/utils.py index 8a185957..c674349b 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -134,7 +134,8 @@ def validate_sql_select(sql): raise InvalidSql(msg) -def path_with_added_args(request, args): +def path_with_added_args(request, args, path=None): + path = path or request.path if isinstance(args, dict): args = args.items() arg_keys = set(a[0] for a in args) @@ -151,7 +152,7 @@ def path_with_added_args(request, args): query_string = urllib.parse.urlencode(sorted(current)) if query_string: query_string = '?{}'.format(query_string) - return request.path + query_string + return path + query_string def path_with_ext(request, ext): diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 5b719b0a..f8f3fa52 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -62,7 +62,7 @@ JSON: ] } -http://localhost:8001/History/downloads.jsono will return that data as +http://localhost:8001/History/downloads.json?_shape=objects will return that data as JSON in a more convenient but less efficient format: :: diff --git a/docs/json_api.rst b/docs/json_api.rst new file mode 100644 index 00000000..6e18f01a --- /dev/null +++ b/docs/json_api.rst @@ -0,0 +1,105 @@ +The Datasette JSON API +====================== + +Datasette provides a JSON API for your SQLite databases. Anything you can do +through the Datasette user interface can also be accessed as JSON via the API. + +To access the API for a page, either click on the ``.json`` link on that page or +edit the URL and add a ``.json`` extension to it. + +If you started Datasette with the ``--cors`` option, each JSON endpoint will be +served with the following additional HTTP header:: + + Access-Control-Allow-Origin: * + +This means JavaScript running on any domain will be able to make cross-origin +requests to fetch the data. + +If you start Datasette without the ``--cors`` option only JavaScript running on +the same domain as Datasette will be able to access the API. + +Different shapes +---------------- + +The default JSON representation of data from a SQLite table or custom query +looks like this:: + + { + "database": "sf-trees", + "table": "qSpecies", + "columns": [ + "id", + "value" + ], + "rows": [ + [ + 1, + "Myoporum laetum :: Myoporum" + ], + [ + 2, + "Metrosideros excelsa :: New Zealand Xmas Tree" + ], + [ + 3, + "Pinus radiata :: Monterey Pine" + ] + ], + "truncated": false, + "next": "100", + "next_url": "http://127.0.0.1:8001/sf-trees-02c8ef1/qSpecies.json?_next=100", + "query_ms": 1.9571781158447266 + } + +The ``columns`` key lists the columns that are being returned, and the ``rows`` +key then returns a list of lists, each one representing a row. The order of the +values in each row corresponds to the columns. + +The ``_shape`` parameter can be used to access alternative formats for the +``rows`` key which may be more convenient for your application. There are three +options: + +* ``?_shape=lists`` - the default option, shown above +* ``?_shape=objects`` - a list of JSON key/value objects +* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows + +``objects`` looks like this:: + + "rows": [ + { + "id": 1, + "value": "Myoporum laetum :: Myoporum" + }, + { + "id": 2, + "value": "Metrosideros excelsa :: New Zealand Xmas Tree" + }, + { + "id": 3, + "value": "Pinus radiata :: Monterey Pine" + } + ] + +``object`` looks like this:: + + "rows": { + "1": { + "id": 1, + "value": "Myoporum laetum :: Myoporum" + }, + "2": { + "id": 2, + "value": "Metrosideros excelsa :: New Zealand Xmas Tree" + }, + "3": { + "id": 3, + "value": "Pinus radiata :: Monterey Pine" + } + ] + +The ``object`` shape is only available for queries against tables - custom SQL +queries and views do not have an obvious primary key so cannot be returned using +this format. + +The ``object`` keys are always strings. If your table has a compound primary +key, the ``object`` keys will be a comma-separated string. diff --git a/tests/test_api.py b/tests/test_api.py index de3cc0c6..733727c6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -118,7 +118,7 @@ def test_database_page(app_client): def test_custom_sql(app_client): response = app_client.get( - '/test_tables.jsono?sql=select+content+from+simple_primary_key', + '/test_tables.json?sql=select+content+from+simple_primary_key&_shape=objects', gather_request=False ) data = response.json @@ -138,7 +138,7 @@ def test_custom_sql(app_client): def test_sql_time_limit(app_client): response = app_client.get( - '/test_tables.jsono?sql=select+sleep(0.5)', + '/test_tables.json?sql=select+sleep(0.5)', gather_request=False ) assert 400 == response.status @@ -147,12 +147,12 @@ def test_sql_time_limit(app_client): def test_custom_sql_time_limit(app_client): response = app_client.get( - '/test_tables.jsono?sql=select+sleep(0.01)', + '/test_tables.json?sql=select+sleep(0.01)', gather_request=False ) assert 200 == response.status response = app_client.get( - '/test_tables.jsono?sql=select+sleep(0.01)&_sql_time_limit_ms=5', + '/test_tables.json?sql=select+sleep(0.01)&_sql_time_limit_ms=5', gather_request=False ) assert 400 == response.status @@ -170,7 +170,7 @@ def test_invalid_custom_sql(app_client): def test_table_json(app_client): - response = app_client.get('/test_tables/simple_primary_key.jsono', gather_request=False) + response = app_client.get('/test_tables/simple_primary_key.json?_shape=objects', gather_request=False) assert response.status == 200 data = response.json assert data['query']['sql'] == 'select * from simple_primary_key order by pk limit 51' @@ -187,8 +187,87 @@ def test_table_json(app_client): }] +def test_jsono_redirects_to_shape_objects(app_client): + response_1 = app_client.get( + '/test_tables/simple_primary_key.jsono', + allow_redirects=False, + gather_request=False + ) + response = app_client.get( + response_1.headers['Location'], + allow_redirects=False, + gather_request=False + ) + assert response.status == 302 + assert response.headers['Location'].endswith('?_shape=objects') + + +def test_table_shape_lists(app_client): + response = app_client.get( + '/test_tables/simple_primary_key.json?_shape=lists', + gather_request=False + ) + assert [ + ['1', 'hello'], + ['2', 'world'], + ['3', ''], + ] == response.json['rows'] + + +def test_table_shape_objects(app_client): + response = app_client.get( + '/test_tables/simple_primary_key.json?_shape=objects', + gather_request=False + ) + assert [{ + 'pk': '1', + 'content': 'hello', + }, { + 'pk': '2', + 'content': 'world', + }, { + 'pk': '3', + 'content': '', + }] == response.json['rows'] + + +def test_table_shape_object(app_client): + response = app_client.get( + '/test_tables/simple_primary_key.json?_shape=object', + gather_request=False + ) + assert { + '1': { + 'pk': '1', + 'content': 'hello', + }, + '2': { + 'pk': '2', + 'content': 'world', + }, + '3': { + 'pk': '3', + 'content': '', + } + } == response.json['rows'] + + +def test_table_shape_object_compound_primary_Key(app_client): + response = app_client.get( + '/test_tables/compound_primary_key.json?_shape=object', + gather_request=False + ) + assert { + 'a,b': { + 'pk1': 'a', + 'pk2': 'b', + 'content': 'c', + } + } == response.json['rows'] + + def test_table_with_slashes_in_name(app_client): - response = app_client.get('/test_tables/table%2Fwith%2Fslashes.csv.jsono', gather_request=False) + response = app_client.get('/test_tables/table%2Fwith%2Fslashes.csv.json?_shape=objects', gather_request=False) assert response.status == 200 data = response.json assert data['rows'] == [{ @@ -198,7 +277,7 @@ def test_table_with_slashes_in_name(app_client): def test_table_with_reserved_word_name(app_client): - response = app_client.get('/test_tables/select.jsono', gather_request=False) + response = app_client.get('/test_tables/select.json?_shape=objects', gather_request=False) assert response.status == 200 data = response.json assert data['rows'] == [{ @@ -210,9 +289,9 @@ def test_table_with_reserved_word_name(app_client): @pytest.mark.parametrize('path,expected_rows,expected_pages', [ - ('/test_tables/no_primary_key.jsono', 201, 5), - ('/test_tables/paginated_view.jsono', 201, 5), - ('/test_tables/123_starts_with_digits.jsono', 0, 1), + ('/test_tables/no_primary_key.json', 201, 5), + ('/test_tables/paginated_view.json', 201, 5), + ('/test_tables/123_starts_with_digits.json', 0, 1), ]) def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages): fetched = [] @@ -232,7 +311,7 @@ def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pag def test_paginate_compound_keys(app_client): fetched = [] - path = '/test_tables/compound_three_primary_keys.jsono' + path = '/test_tables/compound_three_primary_keys.json?_shape=objects' page = 0 while path: page += 1 @@ -250,7 +329,7 @@ def test_paginate_compound_keys(app_client): def test_paginate_compound_keys_with_extra_filters(app_client): fetched = [] - path = '/test_tables/compound_three_primary_keys.jsono?content__contains=d' + path = '/test_tables/compound_three_primary_keys.json?content__contains=d&_shape=objects' page = 0 while path: page += 1 @@ -289,7 +368,7 @@ def test_table_filter_queries(app_client, path, expected_rows): def test_max_returned_rows(app_client): response = app_client.get( - '/test_tables.jsono?sql=select+content+from+no_primary_key', + '/test_tables.json?sql=select+content+from+no_primary_key', gather_request=False ) data = response.json @@ -302,7 +381,7 @@ def test_max_returned_rows(app_client): def test_view(app_client): - response = app_client.get('/test_tables/simple_view.jsono', gather_request=False) + response = app_client.get('/test_tables/simple_view.json?_shape=objects', gather_request=False) assert response.status == 200 data = response.json assert data['rows'] == [{ @@ -318,7 +397,7 @@ def test_view(app_client): def test_row(app_client): - response = app_client.get('/test_tables/simple_primary_key/1.jsono', gather_request=False) + response = app_client.get('/test_tables/simple_primary_key/1.json?_shape=objects', gather_request=False) assert response.status == 200 assert [{'pk': '1', 'content': 'hello'}] == response.json['rows']