2017-07-12 14:06:15 +00:00
|
|
|
DROP TRIGGER IF EXISTS trigger_flag_transportation ON osm_highway_linestring;
|
2017-03-27 20:14:38 +00:00
|
|
|
DROP TRIGGER IF EXISTS trigger_refresh ON transportation.updates;
|
|
|
|
|
|
|
|
-- Instead of using relations to find out the road names we
|
|
|
|
-- stitch together the touching ways with the same name
|
|
|
|
-- to allow for nice label rendering
|
|
|
|
-- Because this works well for roads that do not have relations as well
|
|
|
|
|
2021-11-18 16:35:34 +00:00
|
|
|
-- etldoc: osm_highway_linestring -> osm_transportation_name_network
|
|
|
|
-- etldoc: osm_route_member -> osm_transportation_name_network
|
|
|
|
CREATE TABLE IF NOT EXISTS osm_transportation_name_network AS
|
|
|
|
SELECT
|
|
|
|
geometry,
|
|
|
|
osm_id,
|
|
|
|
tags,
|
|
|
|
ref,
|
|
|
|
highway,
|
|
|
|
subclass,
|
|
|
|
brunnel,
|
|
|
|
"level",
|
|
|
|
sac_scale,
|
|
|
|
layer,
|
|
|
|
indoor,
|
|
|
|
network_type,
|
|
|
|
route_1, route_2, route_3, route_4, route_5, route_6,
|
|
|
|
z_order,
|
|
|
|
route_rank
|
|
|
|
FROM (
|
|
|
|
SELECT DISTINCT ON (hl.osm_id)
|
|
|
|
hl.geometry,
|
|
|
|
hl.osm_id,
|
BUGFIX: Fix name-based way fragmentation in transportation_name (#1295)
I discovered this bug while investigating issues with the updates process related to #1190 #1292, and #814.
The `transportation_name` layer produces slightly different `tags` hstore values in the `osm_transportation_name_linestring` table during the initial import versus when running an update. As currently written, the import code produces null-value keys in the `tags` column, while the update code suppresses them. This PR removes that difference and makes the import code use same method that is currently used in the update code.
With a test case I've written, the import code produces a tags hstore that looks like this:
`"name"=>"OpenMapTiles Secondary 2", "name:de"=>NULL, "name:en"=>NULL, "name_int"=>"OpenMapTiles Secondary 2", "name:latin"=>"OpenMapTiles Secondary 2"`
...while the update code produces a tags hstore that looks like this:
`"name"=>"OpenMapTiles Secondary 2", "name_int"=>"OpenMapTiles Secondary 2", "name:latin"=>"OpenMapTiles Secondary 2"`
Note the missing NULL values.
This bug causes a small amount of space wastage after an update is run, because the update matching code detects the `tags` value as different, resulting in a duplicate copy of the tags value if that row is updated. This causes duplicate objects and breaks GROUP BY clauses that expect to group same-tagged features together. I've tested this by inspection of a generated mbtiles, database spot checks, and the unit test code included in this PR.
2021-11-25 09:45:11 +00:00
|
|
|
transportation_name_tags(hl.geometry, hl.tags, hl.name, hl.name_en, hl.name_de) AS tags,
|
2021-11-18 16:35:34 +00:00
|
|
|
rm1.network_type,
|
|
|
|
CASE
|
|
|
|
WHEN rm1.network_type IS NOT NULL AND rm1.ref::text <> ''
|
|
|
|
THEN rm1.ref::text
|
|
|
|
ELSE NULLIF(hl.ref, '')
|
|
|
|
END AS ref,
|
|
|
|
hl.highway,
|
|
|
|
NULLIF(hl.construction, '') AS subclass,
|
|
|
|
brunnel(hl.is_bridge, hl.is_tunnel, hl.is_ford) AS brunnel,
|
|
|
|
sac_scale,
|
|
|
|
CASE WHEN highway IN ('footway', 'steps') THEN layer END AS layer,
|
|
|
|
CASE WHEN highway IN ('footway', 'steps') THEN level END AS level,
|
|
|
|
CASE WHEN highway IN ('footway', 'steps') THEN indoor END AS indoor,
|
|
|
|
NULLIF(rm1.network, '') || '=' || COALESCE(rm1.ref, '') AS route_1,
|
|
|
|
NULLIF(rm2.network, '') || '=' || COALESCE(rm2.ref, '') AS route_2,
|
|
|
|
NULLIF(rm3.network, '') || '=' || COALESCE(rm3.ref, '') AS route_3,
|
|
|
|
NULLIF(rm4.network, '') || '=' || COALESCE(rm4.ref, '') AS route_4,
|
|
|
|
NULLIF(rm5.network, '') || '=' || COALESCE(rm5.ref, '') AS route_5,
|
|
|
|
NULLIF(rm6.network, '') || '=' || COALESCE(rm6.ref, '') AS route_6,
|
|
|
|
hl.z_order,
|
|
|
|
LEAST(rm1.rank, rm2.rank, rm3.rank, rm4.rank, rm5.rank, rm6.rank) AS route_rank
|
|
|
|
FROM osm_highway_linestring hl
|
|
|
|
LEFT OUTER JOIN osm_route_member rm1 ON rm1.member = hl.osm_id AND rm1.concurrency_index=1
|
|
|
|
LEFT OUTER JOIN osm_route_member rm2 ON rm2.member = hl.osm_id AND rm2.concurrency_index=2
|
|
|
|
LEFT OUTER JOIN osm_route_member rm3 ON rm3.member = hl.osm_id AND rm3.concurrency_index=3
|
|
|
|
LEFT OUTER JOIN osm_route_member rm4 ON rm4.member = hl.osm_id AND rm4.concurrency_index=4
|
|
|
|
LEFT OUTER JOIN osm_route_member rm5 ON rm5.member = hl.osm_id AND rm5.concurrency_index=5
|
|
|
|
LEFT OUTER JOIN osm_route_member rm6 ON rm6.member = hl.osm_id AND rm6.concurrency_index=6
|
|
|
|
WHERE (hl.name <> '' OR hl.ref <> '' OR rm1.ref <> '' OR rm1.network <> '')
|
|
|
|
AND hl.highway <> ''
|
|
|
|
) AS t;
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_network_osm_id_idx ON osm_transportation_name_network (osm_id);
|
BUGFIX: Fix name-based way fragmentation in transportation_name (#1295)
I discovered this bug while investigating issues with the updates process related to #1190 #1292, and #814.
The `transportation_name` layer produces slightly different `tags` hstore values in the `osm_transportation_name_linestring` table during the initial import versus when running an update. As currently written, the import code produces null-value keys in the `tags` column, while the update code suppresses them. This PR removes that difference and makes the import code use same method that is currently used in the update code.
With a test case I've written, the import code produces a tags hstore that looks like this:
`"name"=>"OpenMapTiles Secondary 2", "name:de"=>NULL, "name:en"=>NULL, "name_int"=>"OpenMapTiles Secondary 2", "name:latin"=>"OpenMapTiles Secondary 2"`
...while the update code produces a tags hstore that looks like this:
`"name"=>"OpenMapTiles Secondary 2", "name_int"=>"OpenMapTiles Secondary 2", "name:latin"=>"OpenMapTiles Secondary 2"`
Note the missing NULL values.
This bug causes a small amount of space wastage after an update is run, because the update matching code detects the `tags` value as different, resulting in a duplicate copy of the tags value if that row is updated. This causes duplicate objects and breaks GROUP BY clauses that expect to group same-tagged features together. I've tested this by inspection of a generated mbtiles, database spot checks, and the unit test code included in this PR.
2021-11-25 09:45:11 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_name_network_name_ref_idx ON osm_transportation_name_network (coalesce(tags->'name', ''), coalesce(ref, ''));
|
2021-11-18 16:35:34 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_name_network_geometry_idx ON osm_transportation_name_network USING gist (geometry);
|
2017-03-27 20:14:38 +00:00
|
|
|
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
-- Improve performance of the sql in transportation/update_route_member.sql
|
2017-03-28 14:27:34 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_highway_linestring_highway_partial_idx
|
2020-06-08 16:19:55 +00:00
|
|
|
ON osm_highway_linestring (highway)
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
WHERE highway IN ('motorway', 'trunk');
|
2020-06-08 16:19:55 +00:00
|
|
|
|
2021-04-30 13:39:56 +00:00
|
|
|
-- etldoc: osm_highway_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z11
|
|
|
|
DROP MATERIALIZED VIEW IF EXISTS osm_transportation_merge_linestring_gen_z11 CASCADE;
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z11 AS
|
|
|
|
(
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
SELECT (ST_Dump(ST_LineMerge(ST_Collect(geometry)))).geom AS geometry,
|
2021-04-30 13:39:56 +00:00
|
|
|
NULL::bigint AS osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2021-04-30 13:39:56 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
2021-11-26 06:55:20 +00:00
|
|
|
expressway,
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
min(z_order) as z_order,
|
2021-04-30 13:39:56 +00:00
|
|
|
bicycle,
|
|
|
|
foot,
|
|
|
|
horse,
|
|
|
|
mtb_scale,
|
2021-11-18 16:35:34 +00:00
|
|
|
sac_scale,
|
2021-08-14 05:04:17 +00:00
|
|
|
CASE
|
2021-08-19 05:12:10 +00:00
|
|
|
WHEN access IN ('private', 'no') THEN 'no'
|
|
|
|
ELSE NULL::text END AS access,
|
2021-10-13 18:43:14 +00:00
|
|
|
toll,
|
2021-04-30 13:39:56 +00:00
|
|
|
layer
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
FROM osm_highway_linestring_gen_z11
|
|
|
|
-- mapping.yaml pre-filter: motorway/trunk/primary/secondary/tertiary, with _link variants, construction, ST_IsValid()
|
2021-11-26 06:55:20 +00:00
|
|
|
GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer
|
2021-04-30 13:39:56 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z11_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z11 USING gist (geometry);
|
|
|
|
|
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z10
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z10 AS
|
|
|
|
(
|
|
|
|
SELECT ST_Simplify(geometry, ZRes(12)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2021-04-30 13:39:56 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
2021-11-26 06:55:20 +00:00
|
|
|
expressway,
|
2021-04-30 13:39:56 +00:00
|
|
|
z_order,
|
|
|
|
bicycle,
|
|
|
|
foot,
|
|
|
|
horse,
|
|
|
|
mtb_scale,
|
2021-11-18 16:35:34 +00:00
|
|
|
sac_scale,
|
2021-08-14 05:04:17 +00:00
|
|
|
access,
|
2021-08-19 05:12:10 +00:00
|
|
|
toll,
|
2021-04-30 13:39:56 +00:00
|
|
|
layer
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z11
|
2021-11-01 07:40:07 +00:00
|
|
|
WHERE highway NOT IN ('tertiary', 'tertiary_link', 'busway')
|
|
|
|
AND construction NOT IN ('tertiary', 'tertiary_link', 'busway')
|
2021-04-30 13:39:56 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z10_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z10 USING gist (geometry);
|
|
|
|
|
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z10 -> osm_transportation_merge_linestring_gen_z9
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z9 AS
|
|
|
|
(
|
|
|
|
SELECT ST_Simplify(geometry, ZRes(11)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2021-04-30 13:39:56 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
2021-11-26 06:55:20 +00:00
|
|
|
expressway,
|
2021-04-30 13:39:56 +00:00
|
|
|
z_order,
|
|
|
|
bicycle,
|
|
|
|
foot,
|
|
|
|
horse,
|
|
|
|
mtb_scale,
|
2021-11-18 16:35:34 +00:00
|
|
|
sac_scale,
|
2021-08-14 05:04:17 +00:00
|
|
|
access,
|
2021-08-19 05:12:10 +00:00
|
|
|
toll,
|
2021-04-30 13:39:56 +00:00
|
|
|
layer
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z10
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
-- Current view: motorway/primary/secondary, with _link variants and construction
|
2021-04-30 13:39:56 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z9_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z9 USING gist (geometry);
|
|
|
|
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z9 -> osm_transportation_merge_linestring_gen_z8
|
2021-08-11 08:46:17 +00:00
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z8 AS
|
2020-06-08 16:19:55 +00:00
|
|
|
(
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
SELECT ST_Simplify(ST_LineMerge(ST_Collect(geometry)), ZRes(10)) AS geometry,
|
2020-07-22 11:48:25 +00:00
|
|
|
NULL::bigint AS osm_id,
|
2020-06-08 16:19:55 +00:00
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2020-06-08 16:19:55 +00:00
|
|
|
construction,
|
2020-11-11 13:37:09 +00:00
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
2021-11-26 06:55:20 +00:00
|
|
|
expressway,
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
min(z_order) as z_order
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z9
|
|
|
|
WHERE (highway IN ('motorway', 'trunk', 'primary') OR
|
|
|
|
construction IN ('motorway', 'trunk', 'primary'))
|
|
|
|
AND ST_IsValid(geometry)
|
2021-08-14 05:04:17 +00:00
|
|
|
AND access IS NULL
|
2021-11-26 06:55:20 +00:00
|
|
|
GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
|
2020-06-08 16:19:55 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
2020-11-26 12:54:00 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z8_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z8 USING gist (geometry);
|
2017-03-27 20:14:38 +00:00
|
|
|
|
2020-11-26 12:54:00 +00:00
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z7
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z7 AS
|
2020-06-08 16:19:55 +00:00
|
|
|
(
|
2020-11-26 12:54:00 +00:00
|
|
|
SELECT ST_Simplify(geometry, ZRes(9)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2020-11-26 12:54:00 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
2021-11-26 06:55:20 +00:00
|
|
|
expressway,
|
2020-11-26 12:54:00 +00:00
|
|
|
z_order
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z8
|
2021-08-10 15:55:16 +00:00
|
|
|
-- Current view: motorway/trunk/primary
|
|
|
|
WHERE ST_Length(geometry) > 50
|
2020-06-08 16:19:55 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
2020-11-26 12:54:00 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z7_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z7 USING gist (geometry);
|
2017-03-29 18:55:31 +00:00
|
|
|
|
2020-11-26 12:54:00 +00:00
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z7 -> osm_transportation_merge_linestring_gen_z6
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z6 AS
|
2020-06-08 16:19:55 +00:00
|
|
|
(
|
2020-11-26 12:54:00 +00:00
|
|
|
SELECT ST_Simplify(geometry, ZRes(8)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2020-11-26 12:54:00 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
|
|
|
z_order
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z7
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
WHERE (highway IN ('motorway', 'trunk') OR construction IN ('motorway', 'trunk'))
|
2020-06-08 16:19:55 +00:00
|
|
|
AND ST_Length(geometry) > 100
|
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
2020-11-26 12:54:00 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z6_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z6 USING gist (geometry);
|
2017-03-27 20:14:38 +00:00
|
|
|
|
2020-11-26 12:54:00 +00:00
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z6 -> osm_transportation_merge_linestring_gen_z5
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z5 AS
|
2020-06-08 16:19:55 +00:00
|
|
|
(
|
2020-11-26 12:54:00 +00:00
|
|
|
SELECT ST_Simplify(geometry, ZRes(7)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2020-11-26 12:54:00 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
|
|
|
z_order
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z6
|
2021-08-10 15:55:16 +00:00
|
|
|
WHERE ST_Length(geometry) > 500
|
|
|
|
-- Current view: motorway/trunk
|
2020-06-08 16:19:55 +00:00
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
2020-11-26 12:54:00 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z5_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z5 USING gist (geometry);
|
2017-03-29 18:49:20 +00:00
|
|
|
|
2020-11-26 12:54:00 +00:00
|
|
|
-- etldoc: osm_transportation_merge_linestring_gen_z5 -> osm_transportation_merge_linestring_gen_z4
|
|
|
|
CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z4 AS
|
2020-06-08 16:19:55 +00:00
|
|
|
(
|
2020-11-26 12:54:00 +00:00
|
|
|
SELECT ST_Simplify(geometry, ZRes(6)) AS geometry,
|
|
|
|
osm_id,
|
|
|
|
highway,
|
2021-08-04 08:10:16 +00:00
|
|
|
network,
|
2020-11-26 12:54:00 +00:00
|
|
|
construction,
|
|
|
|
is_bridge,
|
|
|
|
is_tunnel,
|
|
|
|
is_ford,
|
|
|
|
z_order
|
|
|
|
FROM osm_transportation_merge_linestring_gen_z5
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
WHERE (highway = 'motorway' OR construction = 'motorway')
|
2020-06-08 16:19:55 +00:00
|
|
|
AND ST_Length(geometry) > 1000
|
|
|
|
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
|
2020-11-26 12:54:00 +00:00
|
|
|
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z4_geometry_idx
|
|
|
|
ON osm_transportation_merge_linestring_gen_z4 USING gist (geometry);
|
2017-03-27 20:14:38 +00:00
|
|
|
|
|
|
|
|
|
|
|
-- Handle updates
|
|
|
|
|
|
|
|
CREATE SCHEMA IF NOT EXISTS transportation;
|
|
|
|
|
2020-06-08 16:19:55 +00:00
|
|
|
CREATE TABLE IF NOT EXISTS transportation.updates
|
|
|
|
(
|
|
|
|
id serial PRIMARY KEY,
|
2020-07-22 11:48:25 +00:00
|
|
|
t text,
|
2020-06-08 16:19:55 +00:00
|
|
|
UNIQUE (t)
|
|
|
|
);
|
|
|
|
CREATE OR REPLACE FUNCTION transportation.flag() RETURNS trigger AS
|
|
|
|
$$
|
2017-03-27 20:14:38 +00:00
|
|
|
BEGIN
|
2020-06-08 16:19:55 +00:00
|
|
|
INSERT INTO transportation.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
|
|
|
|
RETURN NULL;
|
2017-03-27 20:14:38 +00:00
|
|
|
END;
|
2020-06-08 16:19:55 +00:00
|
|
|
$$ LANGUAGE plpgsql;
|
2017-03-27 20:14:38 +00:00
|
|
|
|
|
|
|
CREATE OR REPLACE FUNCTION transportation.refresh() RETURNS trigger AS
|
2020-06-08 16:19:55 +00:00
|
|
|
$$
|
2020-10-08 13:57:16 +00:00
|
|
|
DECLARE
|
|
|
|
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
|
2020-06-08 16:19:55 +00:00
|
|
|
BEGIN
|
2020-10-08 13:57:16 +00:00
|
|
|
RAISE LOG 'Refresh transportation';
|
Transportation generalization table optimization and cleanup (#1172)
This PR updates the `transportation` layer creation scripts to simplify the SQL and remove unneeded sub-selects, checks and conditionals, fix indexes, and improve inline documentation.
Currently, there are two sequences of materialized view creations. There is one from zoom 11 through 9, and a second one for zoom 8 through 4. This PR removes that break in the sequence of transportation table materialized view creations, in favor of one in which high-zoom views are created first, and then each lower zoom is created from the zoom above.
Instead, the current generalized zoom 8 transportation table is built directly from `osm_transportation_linestring` rather than being built from the zoom 9 transportation table. This means that when building the zoom 8 table, it must scan the entire transportation network rather than just selecting from the pre-filtered zoom 9 table.
This PR removes an unneeded sub-select in the build of the zoom 8 table, which appears to be a leftover from an old version of the SQL that did some sort of merge. Once this PR is implemented all zooms from 11 through 4 will be linked via a progressive series of materialized views.
Lastly, this adds in missing materialized view refreshes for zooms 9-11, which appear to have been entirely missing, and as far as I can tell aren't getting updated in the current version of this code.
In addition, the following optimizations were added as part of this commit:
1. Updates the `osm_highway_linestring_highway_partial_idx` partial index to match the `SELECT..WHERE` clause actually present in `transportation/update_route_member.sql`, which is where it appears to be actually used, and update inline documentation to reflect this.
2. Collapses unnecessary sub-select block in `osm_transportation_merge_linestring_gen_z11`, and removes unnecessary ST_IsValid() call, which already provided in `mapping.yaml`, and update inline documentation to reflect these assumptions.
3. Updates `WHERE` blocks to remove unnecesary checks and further document assumptions. The `highway=construction` check is unnecessary in all cases, because it is sufficient to check the `construction` key alone. If `construction=*` is set, then `highway=construction` is implied.
4. Two indexes were added to `layers/transportation/update_route_member.sql` to improve route population performance.
In testing locally, I'm seeing performance improvements around 10% in the generation of the `transportation` layer, based on modifying `openmaptiles.yaml` to generate only the transportation layer and then repeatedly running `time make import-sql`, however, this timing might be impacted by docker, so I would ask for confirmation of acceptable performance.
In addition, this PR shortens the length of the transportation update SQL file by 30 lines, which makes it easier for contributors to work with.
2021-08-13 16:00:33 +00:00
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z11;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z10;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z9;
|
2020-11-26 12:54:00 +00:00
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z8;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z7;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z6;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z5;
|
|
|
|
REFRESH MATERIALIZED VIEW osm_transportation_merge_linestring_gen_z4;
|
2020-06-08 16:19:55 +00:00
|
|
|
-- noinspection SqlWithoutWhere
|
2017-03-27 20:14:38 +00:00
|
|
|
DELETE FROM transportation.updates;
|
2020-10-08 13:57:16 +00:00
|
|
|
|
|
|
|
RAISE LOG 'Refresh transportation done in %', age(clock_timestamp(), t);
|
2020-06-08 16:19:55 +00:00
|
|
|
RETURN NULL;
|
|
|
|
END;
|
|
|
|
$$ LANGUAGE plpgsql;
|
2017-03-27 20:14:38 +00:00
|
|
|
|
2017-07-12 14:06:15 +00:00
|
|
|
CREATE TRIGGER trigger_flag_transportation
|
2020-06-08 16:19:55 +00:00
|
|
|
AFTER INSERT OR UPDATE OR DELETE
|
|
|
|
ON osm_highway_linestring
|
2017-03-27 20:14:38 +00:00
|
|
|
FOR EACH STATEMENT
|
2020-06-08 16:19:55 +00:00
|
|
|
EXECUTE PROCEDURE transportation.flag();
|
2017-03-27 20:14:38 +00:00
|
|
|
|
|
|
|
CREATE CONSTRAINT TRIGGER trigger_refresh
|
2020-06-08 16:19:55 +00:00
|
|
|
AFTER INSERT
|
|
|
|
ON transportation.updates
|
2017-03-27 20:14:38 +00:00
|
|
|
INITIALLY DEFERRED
|
|
|
|
FOR EACH ROW
|
2021-04-30 13:39:56 +00:00
|
|
|
EXECUTE PROCEDURE transportation.refresh();
|