From 234230e59574ccb8d8a24c45ccd325f725812377 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Dec 2022 06:52:47 -0800 Subject: [PATCH] Default JSON shape is now objects - refs #1914, #1709 --- datasette/renderer.py | 4 +- docs/json_api.rst | 79 ++++++++++++++++++------------------ tests/test_api.py | 2 +- tests/test_canned_queries.py | 4 +- tests/test_plugins.py | 4 +- tests/test_table_api.py | 56 ++++++++++++++----------- 6 files changed, 79 insertions(+), 70 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 45089498..16990efa 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -44,10 +44,10 @@ def json_renderer(args, data, view_name): data["rows"] = [remove_infinites(row) for row in data["rows"]] # Deal with the _shape option - shape = args.get("_shape", "arrays") + shape = args.get("_shape", "objects") # if there's an error, ignore the shape entirely if data.get("error"): - shape = "arrays" + shape = "objects" next_url = data.get("next_url") diff --git a/docs/json_api.rst b/docs/json_api.rst index f00c1fac..b1bafcb1 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -24,46 +24,6 @@ looks like this:: "id", "value" ], - "rows": [ - [ - 1, - "Myoporum laetum :: Myoporum" - ], - [ - 2, - "Metrosideros excelsa :: New Zealand Xmas Tree" - ], - [ - 3, - "Pinus radiata :: Monterey Pine" - ] - ], - "truncated": false, - "next": "100", - "next_url": "http://127.0.0.1:8001/sf-trees-02c8ef1/qSpecies.json?_next=100", - "query_ms": 1.9571781158447266 - } - -The ``columns`` key lists the columns that are being returned, and the ``rows`` -key then returns a list of lists, each one representing a row. The order of the -values in each row corresponds to the columns. - -The ``_shape`` parameter can be used to access alternative formats for the -``rows`` key which may be more convenient for your application. There are three -options: - -* ``?_shape=arrays`` - ``"rows"`` is the default option, shown above -* ``?_shape=objects`` - ``"rows"`` is a list of JSON key/value objects -* ``?_shape=array`` - an JSON array of objects -* ``?_shape=array&_nl=on`` - a newline-separated list of JSON objects -* ``?_shape=arrayfirst`` - a flat JSON array containing just the first value from each row -* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows - -``_shape=objects`` looks like this:: - - { - "database": "sf-trees", - ... "rows": [ { "id": 1, @@ -77,6 +37,45 @@ options: "id": 3, "value": "Pinus radiata :: Monterey Pine" } + ], + "truncated": false, + "next": "100", + "next_url": "http://127.0.0.1:8001/sf-trees-02c8ef1/qSpecies.json?_next=100", + "query_ms": 1.9571781158447266 + } + +The ``columns`` key lists the columns that are being returned, and the ``rows`` +key then returns a list of objets, each one representing a row. + +The ``_shape`` parameter can be used to access alternative formats for the +``rows`` key which may be more convenient for your application. There are three +options: + +* ``?_shape=objects`` - ``"rows"`` is a list of JSON key/value objects - the default +* ``?_shape=arrays`` - ``"rows"`` is a list of lists, where the order of values in each list matches the order of the columns +* ``?_shape=array`` - a JSON array of objects - effectively just the ``"rows"`` key from the default representation +* ``?_shape=array&_nl=on`` - a newline-separated list of JSON objects +* ``?_shape=arrayfirst`` - a flat JSON array containing just the first value from each row +* ``?_shape=object`` - a JSON object keyed using the primary keys of the rows + +``_shape=arrays`` looks like this:: + + { + "database": "sf-trees", + ... + "rows": [ + [ + 1, + "Myoporum laetum :: Myoporum" + ], + [ + 2, + "Metrosideros excelsa :: New Zealand Xmas Tree" + ], + [ + 3, + "Pinus radiata :: Monterey Pine" + ] ] } diff --git a/tests/test_api.py b/tests/test_api.py index 48bca8b3..0e2a55a3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -890,7 +890,7 @@ async def test_json_columns(ds_client, extra_args, expected): def test_config_cache_size(app_client_larger_cache_size): response = app_client_larger_cache_size.get("/fixtures/pragma_cache_size.json") - assert [[-2500]] == response.json["rows"] + assert response.json["rows"] == [{"cache_size": -2500}] def test_config_force_https_urls(): diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index fc70020a..d6a88733 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -75,7 +75,9 @@ def canned_write_immutable_client(): @pytest.mark.asyncio async def test_canned_query_with_named_parameter(ds_client): - response = await ds_client.get("/fixtures/neighborhood_search.json?text=town") + response = await ds_client.get( + "/fixtures/neighborhood_search.json?text=town&_shape=arrays" + ) assert response.json()["rows"] == [ ["Corktown", "Detroit", "MI"], ["Downtown", "Los Angeles", "CA"], diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5077b341..6a576fbd 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -45,9 +45,9 @@ def test_plugin_hooks_have_tests(plugin_hook): @pytest.mark.asyncio async def test_hook_plugins_dir_plugin_prepare_connection(ds_client): response = await ds_client.get( - "/fixtures.json?sql=select+convert_units(100%2C+'m'%2C+'ft')" + "/fixtures.json?_shape=arrayfirst&sql=select+convert_units(100%2C+'m'%2C+'ft')" ) - assert pytest.approx(328.0839) == response.json()["rows"][0][0] + assert response.json()[0] == pytest.approx(328.0839) @pytest.mark.asyncio diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 4562c931..811d0c68 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -380,7 +380,7 @@ async def test_sortable_columns_metadata(ds_client): "path,expected_rows", [ ( - "/fixtures/searchable.json?_search=dog", + "/fixtures/searchable.json?_shape=arrays&_search=dog", [ [1, "barry cat", "terry dog", "panther"], [2, "terry dog", "sara weasel", "puma"], @@ -388,17 +388,17 @@ async def test_sortable_columns_metadata(ds_client): ), ( # Special keyword shouldn't break FTS query - "/fixtures/searchable.json?_search=AND", + "/fixtures/searchable.json?_shape=arrays&_search=AND", [], ), ( # Without _searchmode=raw this should return no results - "/fixtures/searchable.json?_search=te*+AND+do*", + "/fixtures/searchable.json?_shape=arrays&_search=te*+AND+do*", [], ), ( # _searchmode=raw - "/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw", + "/fixtures/searchable.json?_shape=arrays&_search=te*+AND+do*&_searchmode=raw", [ [1, "barry cat", "terry dog", "panther"], [2, "terry dog", "sara weasel", "puma"], @@ -406,21 +406,21 @@ async def test_sortable_columns_metadata(ds_client): ), ( # _searchmode=raw combined with _search_COLUMN - "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw", + "/fixtures/searchable.json?_shape=arrays&_search_text2=te*&_searchmode=raw", [ [1, "barry cat", "terry dog", "panther"], ], ), ( - "/fixtures/searchable.json?_search=weasel", + "/fixtures/searchable.json?_shape=arrays&_search=weasel", [[2, "terry dog", "sara weasel", "puma"]], ), ( - "/fixtures/searchable.json?_search_text2=dog", + "/fixtures/searchable.json?_shape=arrays&_search_text2=dog", [[1, "barry cat", "terry dog", "panther"]], ), ( - "/fixtures/searchable.json?_search_name%20with%20.%20and%20spaces=panther", + "/fixtures/searchable.json?_shape=arrays&_search_name%20with%20.%20and%20spaces=panther", [[1, "barry cat", "terry dog", "panther"]], ), ], @@ -466,7 +466,7 @@ def test_searchmode(table_metadata, querystring, expected_rows): with make_app_client( metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}} ) as client: - response = client.get("/fixtures/searchable.json?" + querystring) + response = client.get("/fixtures/searchable.json?_shape=arrays&" + querystring) assert expected_rows == response.json["rows"] @@ -475,26 +475,26 @@ def test_searchmode(table_metadata, querystring, expected_rows): "path,expected_rows", [ ( - "/fixtures/searchable_view_configured_by_metadata.json?_search=weasel", + "/fixtures/searchable_view_configured_by_metadata.json?_shape=arrays&_search=weasel", [[2, "terry dog", "sara weasel", "puma"]], ), # This should return all results because search is not configured: ( - "/fixtures/searchable_view.json?_search=weasel", + "/fixtures/searchable_view.json?_shape=arrays&_search=weasel", [ [1, "barry cat", "terry dog", "panther"], [2, "terry dog", "sara weasel", "puma"], ], ), ( - "/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk", + "/fixtures/searchable_view.json?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk", [[2, "terry dog", "sara weasel", "puma"]], ), ], ) async def test_searchable_views(ds_client, path, expected_rows): response = await ds_client.get(path) - assert expected_rows == response.json()["rows"] + assert response.json()["rows"] == expected_rows @pytest.mark.asyncio @@ -513,18 +513,24 @@ async def test_searchable_invalid_column(ds_client): @pytest.mark.parametrize( "path,expected_rows", [ - ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]), ( - "/fixtures/simple_primary_key.json?content__contains=o", + "/fixtures/simple_primary_key.json?_shape=arrays&content=hello", + [["1", "hello"]], + ), + ( + "/fixtures/simple_primary_key.json?_shape=arrays&content__contains=o", [ ["1", "hello"], ["2", "world"], ["4", "RENDER_CELL_DEMO"], ], ), - ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]), ( - "/fixtures/simple_primary_key.json?content__not=world", + "/fixtures/simple_primary_key.json?_shape=arrays&content__exact=", + [["3", ""]], + ), + ( + "/fixtures/simple_primary_key.json?_shape=arrays&content__not=world", [ ["1", "hello"], ["3", ""], @@ -536,13 +542,13 @@ async def test_searchable_invalid_column(ds_client): ) async def test_table_filter_queries(ds_client, path, expected_rows): response = await ds_client.get(path) - assert expected_rows == response.json()["rows"] + assert response.json()["rows"] == expected_rows @pytest.mark.asyncio async def test_table_filter_queries_multiple_of_same_type(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key.json?content__not=world&content__not=hello" + "/fixtures/simple_primary_key.json?_shape=arrays&content__not=world&content__not=hello" ) assert [ ["3", ""], @@ -554,7 +560,9 @@ async def test_table_filter_queries_multiple_of_same_type(ds_client): @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") @pytest.mark.asyncio async def test_table_filter_json_arraycontains(ds_client): - response = await ds_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") + response = await ds_client.get( + "/fixtures/facetable.json?_shape=arrays&tags__arraycontains=tag1" + ) assert response.json()["rows"] == [ [ 1, @@ -589,7 +597,7 @@ async def test_table_filter_json_arraycontains(ds_client): @pytest.mark.asyncio async def test_table_filter_json_arraynotcontains(ds_client): response = await ds_client.get( - "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]" + "/fixtures/facetable.json?_shape=arrays&tags__arraynotcontains=tag3&tags__not=[]" ) assert response.json()["rows"] == [ [ @@ -611,7 +619,7 @@ async def test_table_filter_json_arraynotcontains(ds_client): @pytest.mark.asyncio async def test_table_filter_extra_where(ds_client): response = await ds_client.get( - "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'" + "/fixtures/facetable.json?_shape=arrays&_where=_neighborhood='Dogpatch'" ) assert [ [ @@ -652,7 +660,7 @@ def test_table_filter_extra_where_disabled_if_no_sql_allowed(): async def test_table_through(ds_client): # Just the museums: response = await ds_client.get( - '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + '/fixtures/roadside_attractions.json?_shape=arrays&_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' ) assert response.json()["rows"] == [ [ @@ -707,7 +715,7 @@ async def test_view(ds_client): @pytest.mark.asyncio async def test_unit_filters(ds_client): response = await ds_client.get( - "/fixtures/units.json?distance__lt=75km&frequency__gt=1kHz" + "/fixtures/units.json?_shape=arrays&distance__lt=75km&frequency__gt=1kHz" ) assert response.status_code == 200 data = response.json()