2016-11-28 19:16:27 +00:00
|
|
|
-- etldoc: layer_park[shape=record fillcolor=lightpink, style="rounded,filled",
|
2016-12-01 08:29:34 +00:00
|
|
|
-- etldoc: label="layer_park |<z6> z6 |<z7> z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14> z14+" ] ;
|
2016-11-28 19:16:27 +00:00
|
|
|
|
2018-11-07 17:18:11 +00:00
|
|
|
CREATE OR REPLACE FUNCTION layer_park(bbox geometry, zoom_level int, pixel_width numeric)
|
|
|
|
RETURNS TABLE(osm_id bigint, geometry geometry, class text, name text, name_en text, name_de text, tags hstore, rank int) AS $$
|
|
|
|
SELECT osm_id, geometry, class, name, name_en, name_de, tags, rank
|
|
|
|
FROM (
|
2016-11-28 19:16:27 +00:00
|
|
|
SELECT osm_id, geometry,
|
2020-01-27 09:12:57 +00:00
|
|
|
COALESCE(
|
|
|
|
LOWER(REPLACE(NULLIF(protection_title, ''), ' ', '_')),
|
|
|
|
NULLIF(boundary, ''),
|
|
|
|
NULLIF(leisure, '')
|
|
|
|
) AS class,
|
2018-11-07 17:18:11 +00:00
|
|
|
name, name_en, name_de, tags,
|
|
|
|
NULL::int as rank
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM (
|
2016-11-30 17:11:36 +00:00
|
|
|
-- etldoc: osm_park_polygon_gen8 -> layer_park:z6
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-30 18:37:04 +00:00
|
|
|
FROM osm_park_polygon_gen8
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 6 AND geometry && bbox
|
2016-11-30 17:11:36 +00:00
|
|
|
UNION ALL
|
2016-11-30 14:15:59 +00:00
|
|
|
-- etldoc: osm_park_polygon_gen7 -> layer_park:z7
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-30 14:15:59 +00:00
|
|
|
FROM osm_park_polygon_gen7
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 7 AND geometry && bbox
|
2016-11-30 14:15:59 +00:00
|
|
|
UNION ALL
|
2016-11-28 19:16:27 +00:00
|
|
|
-- etldoc: osm_park_polygon_gen6 -> layer_park:z8
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen6
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 8 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon_gen5 -> layer_park:z9
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen5
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 9 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon_gen4 -> layer_park:z10
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen4
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 10 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon_gen3 -> layer_park:z11
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen3
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 11 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon_gen2 -> layer_park:z12
|
2020-04-04 09:50:03 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen2
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 12 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon_gen1 -> layer_park:z13
|
2020-04-04 10:16:59 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon_gen1
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level = 13 AND geometry && bbox
|
2016-11-28 19:16:27 +00:00
|
|
|
UNION ALL
|
|
|
|
-- etldoc: osm_park_polygon -> layer_park:z14
|
2020-04-04 10:16:59 +00:00
|
|
|
SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title
|
2016-11-28 19:16:27 +00:00
|
|
|
FROM osm_park_polygon
|
2018-11-23 15:56:20 +00:00
|
|
|
WHERE zoom_level >= 14 AND geometry && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
) AS park_polygon
|
|
|
|
|
|
|
|
UNION ALL
|
|
|
|
SELECT osm_id, geometry_point AS geometry,
|
2020-01-27 09:12:57 +00:00
|
|
|
COALESCE(
|
|
|
|
LOWER(REPLACE(NULLIF(protection_title, ''), ' ', '_')),
|
|
|
|
NULLIF(boundary, ''),
|
|
|
|
NULLIF(leisure, '')
|
|
|
|
) AS class,
|
2018-11-07 17:18:11 +00:00
|
|
|
name, name_en, name_de, tags,
|
|
|
|
row_number() OVER (
|
|
|
|
PARTITION BY LabelGrid(geometry_point, 100 * pixel_width)
|
|
|
|
ORDER BY
|
|
|
|
(CASE WHEN boundary = 'national_park' THEN true ELSE false END) DESC,
|
|
|
|
(COALESCE(NULLIF(tags->'wikipedia', ''), NULLIF(tags->'wikidata', '')) IS NOT NULL) DESC,
|
|
|
|
area DESC
|
|
|
|
)::int AS "rank"
|
|
|
|
FROM (
|
|
|
|
-- etldoc: osm_park_polygon_gen8 -> layer_park:z6
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen8
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 6 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen7 -> layer_park:z7
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen7
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 7 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen6 -> layer_park:z8
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen6
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 8 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen5 -> layer_park:z9
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen5
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 9 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen4 -> layer_park:z10
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen4
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 10 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen3 -> layer_park:z11
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen3
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 11 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen2 -> layer_park:z12
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen2
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 12 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon_gen1 -> layer_park:z13
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon_gen1
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level = 13 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
UNION ALL
|
|
|
|
|
|
|
|
-- etldoc: osm_park_polygon -> layer_park:z14
|
2020-01-27 09:12:57 +00:00
|
|
|
SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, protection_title, area
|
2018-11-07 17:18:11 +00:00
|
|
|
FROM osm_park_polygon
|
2018-11-23 14:27:57 +00:00
|
|
|
WHERE zoom_level >= 14 AND geometry_point && bbox
|
2018-11-07 17:18:11 +00:00
|
|
|
) AS park_point
|
2018-11-23 15:56:20 +00:00
|
|
|
) AS park_all;
|
Parallel capability to layer functions (#728)
This PR allows queries to be parallelized on recent versions of Postgres. The `PARALLEL SAFE` modifier has been added to the layer functions and a PLPGSQL function to convert strings into number has been replaced.
`PARALLEL SAFE` is a modifier for `CREATE FUNCTION` available since Postgres 9.6, so this change does not break current OpenMapTiles supported database version. More details about this topic [here](https://www.postgresql.org/docs/current/parallel-safety.html) and at the reference documentation for [`CREATE FUNCTION`](https://www.postgresql.org/docs/current/sql-createfunction.html).
### Testing procedure
The procedure to test this was:
* Imported `spain.pbf` in a clean environment
* Dumped the OpenMapTiles database from the Postgres Docker image
* Created a clean Postgres 12 database using the default Docker image
* Installed `postgis` 3 from the default Debian package and `osml10n` 2.5.8 from the repository (`make`, etc.)
* Restored the dump
* Lowered the postgres planner parameters for triggering parallel plans:
```sql
set parallel_setup_cost = 5;
set parallel_tuple_cost = 0.005;
```
* Manually added the `PARALLEL SAFE` modifier to each function involved in layer queries (not on updates or inserting functions).
* For each layer, run a testing query to confirm parallel workers were created, something like this:
```sql
explain analyze
select * from layer_aerodrome_label(tilebbox(8,128,95),10,null)
union all
select * from layer_aerodrome_label(tilebbox(8,128,97),10,null);
```
* After all the layers were processed and confirmed to start parallel executions, a more complete example was run. This example just retrieves the geometries for all the layers from the same tile but without using any MVT related function.
<details><summary>Testing query</summary>
```sql
-- Using the function layer_landuse
explain analyze
select geometry from layer_water(tilebbox(14,8020,6178),14)
union all
select geometry from layer_waterway(tilebbox(14,8020,6178),14)
union all
select geometry from layer_landcover(tilebbox(14,8020,6178),14)
union all
select geometry from layer_landuse(tilebbox(14,8020,6178),14)
union all
select geometry from layer_mountain_peak(tilebbox(14,8020,6178),14)
union all
select geometry from layer_park(tilebbox(14,8020,6178),14)
union all
select geometry from layer_boundary(tilebbox(14,8020,6178),14)
union all
select geometry from layer_aeroway(tilebbox(14,8020,6178),14)
union all
select geometry from layer_transportation(tilebbox(14,8020,6178),14)
union all
select geometry from layer_building(tilebbox(14,8020,6178),14)
union all
select geometry from layer_water_name(tilebbox(14,8020,6178),14)
union all
select geometry from layer_transportation_name(tilebbox(14,8020,6178),14)
union all
select geometry from layer_place(tilebbox(14,8020,6178),14)
union all
select geometry from layer_housenumber(tilebbox(14,8020,6178),14)
union all
select geometry from layer_poi(tilebbox(14,8020,6178),14)
union all
select geometry from layer_aerodrome_label(tilebbox(14,8020,6178),14);
```
</details>
You can inspect the execution plan and results on [this page](https://explain.dalibo.com/plan/3z). Also [attaching](https://github.com/openmaptiles/openmaptiles/files/3951822/explain-tile-simple.tar.gz) the query and JSON output for future reference. The website gives a ton of details, but you may want to search for nodes mentioning `workers` or `parallel` like in this area referring to `osm_border` or `osm_aeroway_linestring` entities

### Next steps
Since the execution plan is not showing a parallel append at the top level, meaning it's not running each layer individually, I want to continue experimenting with parameters and queries to see if it's possible to even parallelize more the request.
I will post my finding here, even no change in the code should happen.
cc. @nyurik
Co-authored-by: Yuri Astrakhan <yuriastrakhan@gmail.com>
2020-02-01 00:36:02 +00:00
|
|
|
$$
|
|
|
|
LANGUAGE SQL
|
|
|
|
IMMUTABLE PARALLEL SAFE;
|