diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3c4b31f4..18d75c85 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
# Introduction
-Thank you for considering contributing to OpenMapTiles. It's people like you that make OpenMapTiles such a great project. Talk to us at the OSM Slack **#openmaptiles** channel ([join](https://osmus-slack.herokuapp.com/)).
+Thank you for considering contributing to OpenMapTiles. It's people like you that make OpenMapTiles such a great project. Talk to us at the OSM Slack **#openmaptiles** channel ([join](https://slack.openstreetmap.us/)).
Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests.
diff --git a/Makefile b/Makefile
index 0a355897..b7c22a00 100644
--- a/Makefile
+++ b/Makefile
@@ -25,13 +25,23 @@ export TPORT
STYLE_FILE := build/style/style.json
STYLE_HEADER_FILE := style/style-header.json
+# Support newer `docker compose` syntax in addition to `docker-compose`
+
+ifeq (, $(shell which docker-compose))
+ DOCKER_COMPOSE_COMMAND := docker compose
+ $(info Using docker compose V2 (docker compose))
+else
+ DOCKER_COMPOSE_COMMAND := docker-compose
+ $(info Using docker compose V1 (docker-compose))
+endif
+
# Allow a custom docker-compose project name
DC_PROJECT := $(or $(DC_PROJECT),$(shell (. .env; echo $${DC_PROJECT})))
ifeq ($(DC_PROJECT),)
DC_PROJECT := $(notdir $(shell pwd))
- DOCKER_COMPOSE := docker-compose
+ DOCKER_COMPOSE := $(DOCKER_COMPOSE_COMMAND)
else
- DOCKER_COMPOSE := docker-compose --project-name $(DC_PROJECT)
+ DOCKER_COMPOSE := $(DOCKER_COMPOSE_COMMAND) --project-name $(DC_PROJECT)
endif
# Make some operations quieter (e.g. inside the test script)
@@ -489,7 +499,7 @@ start-tileserver: init-dirs build-style download-fonts
@echo "* "
@echo "***********************************************************"
@echo " "
- docker pull maptiler/tileserver-gl
+ $(DOCKER_COMPOSE_COMMAND) pull tileserver-gl
@echo " "
@echo "***********************************************************"
@echo "* "
@@ -606,9 +616,9 @@ else
@echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
ifneq ($(USE_PRELOADED_IMAGE),)
POSTGIS_IMAGE=openmaptiles/postgis-preloaded \
- docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres
+ $(DOCKER_COMPOSE_COMMAND) pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres
else
- docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres import-data
+ $(DOCKER_COMPOSE_COMMAND) pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres import-data
endif
endif
@@ -633,7 +643,7 @@ test-perf-null: init-dirs
.PHONY: build-test-pbf
build-test-pbf: init-dirs
- docker-compose run $(DC_OPTS) openmaptiles-tools /tileset/.github/workflows/build-test-data.sh
+ $(DOCKER_COMPOSE_COMMAND) run $(DC_OPTS) openmaptiles-tools /tileset/.github/workflows/build-test-data.sh
.PHONY: debug
debug: ## Use this target when developing Makefile itself to verify loaded environment variables
@@ -675,11 +685,13 @@ test-sql: clean refresh-docker-images destroy-db start-db-nowait build/import-te
awk '1{print; fflush()} $$0~".*ERROR" {txt=$$0} END{ if(txt){print "\n*** ERROR detected, aborting:"; print txt; exit(1)} }'
@echo "Test SQL output for Import Test Data"
- $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-import.sql'
+ $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-import.sql' 2>&1 | \
+ awk -v s="ERROR:" '1{print; fflush()} $$0~s{print "*** ERROR detected, aborting"; exit(1)}'
@echo "Run UPDATE process on test data..."
sed -ir "s/^[#]*\s*DIFF_MODE=.*/DIFF_MODE=true/" .env
$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-diff'
@echo "Test SQL output for Update Test Data"
- $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-update.sql'
+ $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-update.sql' 2>&1 | \
+ awk -v s="ERROR:" '1{print; fflush()} $$0~s{print "*** ERROR detected, aborting"; exit(1)}'
diff --git a/README.md b/README.md
index 7bccd3b2..ebfb8536 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,8 @@ Please keep in mind that OpenMapTiles schema should display general topographic
- :link: Data for download: https://www.maptiler.com/data/
- :link: Hosting https://www.maptiler.com/cloud/
- :link: Create own layer https://github.com/openmaptiles/openmaptiles-skiing
-- :link: Discuss at the #openmaptiles channel at [OSM Slack](https://osmus-slack.herokuapp.com/)
+- :link: Practical usage of OpenMapTiles https://github.com/maptiler/foss4g-workshop
+- :link: Discuss at the #openmaptiles channel at [OSM Slack](https://slack.openstreetmap.us/)
## Styles
@@ -102,7 +103,7 @@ Now start up the database container.
make start-db
```
-Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). Natural Earth country boundaries are used in the few lowest zoom levels.
+Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/openmaptiles/osm-lakelines). Natural Earth country boundaries are used in the few lowest zoom levels.
```bash
make import-data
diff --git a/integrity.sh b/integrity.sh
new file mode 100755
index 00000000..831f14ae
--- /dev/null
+++ b/integrity.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# A script to run the "integrity" continuous integration script.
+
+area=monaco
+echo MIN_ZOOM=0 >> .env
+echo MAX_ZOOM=14 >> .env
+./quickstart.sh $area
+export TEST_MODE=yes
+make generate-devdoc
+area=europe/monaco
+echo DIFF_MODE=true >> .env
+
+# Cleanup
+rm -fr data build cache
+# Create data/$area.repl.json
+make download-geofabrik area=$area
+# Download 2+ month old data
+export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+echo Downloading $old_date extract of $area
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+# Initial import and tile generation
+./quickstart.sh $area
+sleep 2
+echo Downloading updates
+# Loop to recover from potential "ERROR 429: Too Many Requests"
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do
+ sleep 2;
+ echo Sleeping...;
+ sleep 630;
+done"
+echo Downloading updates completed
+echo Importing updates
+make import-diff
+echo Generating new tiles
+make generate-tiles-pg
diff --git a/layers/aerodrome_label/update_aerodrome_label_point.sql b/layers/aerodrome_label/update_aerodrome_label_point.sql
index 72eb175a..c5d675a8 100644
--- a/layers/aerodrome_label/update_aerodrome_label_point.sql
+++ b/layers/aerodrome_label/update_aerodrome_label_point.sql
@@ -12,7 +12,7 @@ CREATE SCHEMA IF NOT EXISTS aerodrome_label;
CREATE TABLE IF NOT EXISTS aerodrome_label.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_aerodrome_label_point -> osm_aerodrome_label_point
@@ -48,11 +48,7 @@ SELECT update_aerodrome_label_point(true);
CREATE OR REPLACE FUNCTION aerodrome_label.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO aerodrome_label.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO aerodrome_label.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO aerodrome_label.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -77,6 +73,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh aerodrome_label';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE aerodrome_label.osm_ids;
+ ANALYZE osm_aerodrome_label_point;
+
PERFORM update_aerodrome_label_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM aerodrome_label.osm_ids;
@@ -89,13 +90,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_aerodrome_label_point
FOR EACH ROW
EXECUTE PROCEDURE aerodrome_label.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_aerodrome_label_point
FOR EACH STATEMENT
EXECUTE PROCEDURE aerodrome_label.flag();
diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql
index fa73bd16..e95de1f2 100644
--- a/layers/boundary/boundary.sql
+++ b/layers/boundary/boundary.sql
@@ -180,7 +180,7 @@ CREATE MATERIALIZED VIEW osm_border_disp_linestring_gen_z1 AS
SELECT ST_Simplify(geometry, ZRes(2)) AS geometry, adm0_l, adm0_r, admin_level, TRUE AS disputed, name, claimed_by, maritime
FROM osm_border_disp_linestring_gen_z2
) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
-CREATE INDEX IF NOT EXISTS osm_border_disp_linestring_gen_z2_idx ON osm_border_disp_linestring_gen_z2 USING gist (geometry);
+CREATE INDEX IF NOT EXISTS osm_border_disp_linestring_gen_z1_idx ON osm_border_disp_linestring_gen_z1 USING gist (geometry);
-- ne_10m_admin_0_boundary_lines_land
-- etldoc: ne_10m_admin_0_boundary_lines_land -> ne_10m_admin_0_boundary_lines_land_gen_z4
@@ -348,7 +348,8 @@ FROM ne_110m_admin_0_boundary_lines_land_gen_z0
-- etldoc: ne_50m_admin_0_boundary_lines_land_gen_z1 -> boundary_z1
-- etldoc: ne_10m_admin_1_states_provinces_lines_gen_z1 -> boundary_z1
-- etldoc: osm_border_disp_linestring_gen_z1 -> boundary_z1
-CREATE OR REPLACE VIEW boundary_z1 AS
+DROP MATERIALIZED VIEW IF EXISTS boundary_z1 CASCADE;
+CREATE MATERIALIZED VIEW boundary_z1 AS
(
SELECT geometry,
admin_level,
@@ -380,12 +381,14 @@ SELECT geometry,
maritime
FROM osm_border_disp_linestring_gen_z1
);
+CREATE INDEX IF NOT EXISTS boundary_z1_idx ON boundary_z1 USING gist (geometry);
-- etldoc: ne_50m_admin_0_boundary_lines_land_gen_z2 -> boundary_z2
-- etldoc: ne_10m_admin_1_states_provinces_lines_gen_z2 -> boundary_z2
-- etldoc: osm_border_disp_linestring_gen_z2 -> boundary_z2
-CREATE OR REPLACE VIEW boundary_z2 AS
+DROP MATERIALIZED VIEW IF EXISTS boundary_z2 CASCADE;
+CREATE MATERIALIZED VIEW boundary_z2 AS
(
SELECT geometry,
admin_level,
@@ -417,11 +420,13 @@ SELECT geometry,
maritime
FROM osm_border_disp_linestring_gen_z2
);
+CREATE INDEX IF NOT EXISTS boundary_z2_idx ON boundary_z2 USING gist (geometry);
-- etldoc: ne_50m_admin_0_boundary_lines_land_gen_z3 -> boundary_z3
-- etldoc: ne_10m_admin_1_states_provinces_lines_gen_z3 -> boundary_z3
-- etldoc: osm_border_disp_linestring_gen_z3 -> boundary_z3
-CREATE OR REPLACE VIEW boundary_z3 AS
+DROP MATERIALIZED VIEW IF EXISTS boundary_z3 CASCADE;
+CREATE MATERIALIZED VIEW boundary_z3 AS
(
SELECT geometry,
admin_level,
@@ -453,11 +458,13 @@ SELECT geometry,
maritime
FROM osm_border_disp_linestring_gen_z3
);
+CREATE INDEX IF NOT EXISTS boundary_z3_idx ON boundary_z3 USING gist (geometry);
-- etldoc: ne_10m_admin_0_boundary_lines_land_gen_z4 -> boundary_z4
-- etldoc: ne_10m_admin_1_states_provinces_lines_gen_z4 -> boundary_z4
-- etldoc: osm_border_linestring_gen_z4 -> boundary_z4
-CREATE OR REPLACE VIEW boundary_z4 AS
+DROP MATERIALIZED VIEW IF EXISTS boundary_z4 CASCADE;
+CREATE MATERIALIZED VIEW boundary_z4 AS
(
SELECT geometry,
admin_level,
@@ -489,6 +496,7 @@ SELECT geometry,
maritime
FROM osm_border_linestring_gen_z4
);
+CREATE INDEX IF NOT EXISTS boundary_z4_idx ON boundary_z4 USING gist (geometry);
-- etldoc: osm_border_linestring_gen_z5 -> boundary_z5
CREATE OR REPLACE VIEW boundary_z5 AS
diff --git a/layers/housenumber/housenumber.sql b/layers/housenumber/housenumber.sql
index 54cccef5..22dc4385 100644
--- a/layers/housenumber/housenumber.sql
+++ b/layers/housenumber/housenumber.sql
@@ -14,7 +14,7 @@ SELECT
-- etldoc: osm_housenumber_point -> layer_housenumber:z14_
osm_id,
geometry,
- housenumber
+ display_housenumber(housenumber)
FROM (
SELECT
osm_id,
diff --git a/layers/housenumber/housenumber.yaml b/layers/housenumber/housenumber.yaml
index 7fa1e6e0..99a8a7cb 100644
--- a/layers/housenumber/housenumber.yaml
+++ b/layers/housenumber/housenumber.yaml
@@ -7,12 +7,14 @@ layer:
buffer_size: 8
srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over
fields:
- housenumber: Value of the [`addr:housenumber`](http://wiki.openstreetmap.org/wiki/Key:addr) tag.
+ housenumber: Value of the [`addr:housenumber`](http://wiki.openstreetmap.org/wiki/Key:addr) tag.
+ If there are multiple values separated by semi-colons, the first and last value separated by a dash.
datasource:
geometry_field: geometry
srid: 900913
query: (SELECT geometry, housenumber FROM layer_housenumber(!bbox!, z(!scale_denominator!))) AS t
schema:
+ - ./housenumber_display.sql
- ./housenumber_centroid.sql
- ./housenumber.sql
datasources:
diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql
index 66db8a1f..425cd905 100644
--- a/layers/housenumber/housenumber_centroid.sql
+++ b/layers/housenumber/housenumber_centroid.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS housenumber;
CREATE TABLE IF NOT EXISTS housenumber.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_housenumber_point -> osm_housenumber_point
@@ -41,11 +41,7 @@ SELECT convert_housenumber_point(true);
CREATE OR REPLACE FUNCTION housenumber.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO housenumber.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO housenumber.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO housenumber.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -70,6 +66,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh housenumber';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE housenumber.osm_ids;
+ ANALYZE osm_housenumber_point;
+
PERFORM convert_housenumber_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM housenumber.osm_ids;
@@ -82,13 +83,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_housenumber_point
FOR EACH ROW
EXECUTE PROCEDURE housenumber.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_housenumber_point
FOR EACH STATEMENT
EXECUTE PROCEDURE housenumber.flag();
diff --git a/layers/housenumber/housenumber_display.sql b/layers/housenumber/housenumber_display.sql
new file mode 100644
index 00000000..7b4a46b0
--- /dev/null
+++ b/layers/housenumber/housenumber_display.sql
@@ -0,0 +1,20 @@
+CREATE OR REPLACE FUNCTION display_housenumber_nonnumeric(raw_housenumber text)
+RETURNS text AS $$
+ -- Find the position of the semicolon in the input string
+ -- and extract the first and last value
+ SELECT substring(raw_housenumber from 1 for position(';' in raw_housenumber) - 1)
+ || '–'
+ || substring(raw_housenumber from position(';' in raw_housenumber) + 1);
+$$ LANGUAGE SQL IMMUTABLE;
+
+
+CREATE OR REPLACE FUNCTION display_housenumber(raw_housenumber text)
+RETURNS text AS $$
+ SELECT CASE
+ WHEN raw_housenumber !~ ';' THEN raw_housenumber
+ WHEN raw_housenumber ~ '[^0-9;]' THEN display_housenumber_nonnumeric(raw_housenumber)
+ ELSE
+ (SELECT min(value)::text || '–' || max(value)::text
+ FROM unnest(string_to_array(raw_housenumber, ';')::int[]) AS value)
+ END
+$$ LANGUAGE SQL IMMUTABLE;
diff --git a/layers/mountain_peak/update_mountain_linestring.sql b/layers/mountain_peak/update_mountain_linestring.sql
index 760c2742..4862ff8b 100644
--- a/layers/mountain_peak/update_mountain_linestring.sql
+++ b/layers/mountain_peak/update_mountain_linestring.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS mountain_linestring;
CREATE TABLE IF NOT EXISTS mountain_linestring.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_mountain_linestring -> osm_mountain_linestring
@@ -26,11 +26,7 @@ SELECT update_osm_mountain_linestring(true);
CREATE OR REPLACE FUNCTION mountain_linestring.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO mountain_linestring.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO mountain_linestring.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO mountain_linestring.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -55,6 +51,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh mountain_linestring';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE mountain_linestring.osm_ids;
+ ANALYZE osm_mountain_linestring;
+
PERFORM update_osm_mountain_linestring(false);
-- noinspection SqlWithoutWhere
DELETE FROM mountain_linestring.osm_ids;
@@ -67,13 +68,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_mountain_linestring
FOR EACH ROW
EXECUTE PROCEDURE mountain_linestring.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_mountain_linestring
FOR EACH STATEMENT
EXECUTE PROCEDURE mountain_linestring.flag();
diff --git a/layers/mountain_peak/update_peak_point.sql b/layers/mountain_peak/update_peak_point.sql
index cda77f3d..73465387 100644
--- a/layers/mountain_peak/update_peak_point.sql
+++ b/layers/mountain_peak/update_peak_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS mountain_peak_point;
CREATE TABLE IF NOT EXISTS mountain_peak_point.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_peak_point -> osm_peak_point
@@ -26,11 +26,7 @@ SELECT update_osm_peak_point(true);
CREATE OR REPLACE FUNCTION mountain_peak_point.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO mountain_peak_point.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO mountain_peak_point.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO mountain_peak_point.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -55,6 +51,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh mountain_peak_point';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE mountain_peak_point.osm_ids;
+ ANALYZE osm_peak_point;
+
PERFORM update_osm_peak_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM mountain_peak_point.osm_ids;
@@ -67,13 +68,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_peak_point
FOR EACH ROW
EXECUTE PROCEDURE mountain_peak_point.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_peak_point
FOR EACH STATEMENT
EXECUTE PROCEDURE mountain_peak_point.flag();
diff --git a/layers/park/mapping.yaml b/layers/park/mapping.yaml
index 8d3b9e85..a905a9b1 100644
--- a/layers/park/mapping.yaml
+++ b/layers/park/mapping.yaml
@@ -93,6 +93,9 @@ tables:
- name: historic
key: historic
type: string
+ - name: seamark_type
+ key: seamark:type
+ type: string
- name: area
type: area
mapping:
@@ -101,6 +104,7 @@ tables:
boundary:
- national_park
- protected_area
+ - aboriginal_lands
historic:
- archaeological_site
- battlefield
diff --git a/layers/park/park.sql b/layers/park/park.sql
index f92b3ec0..5bbb396f 100644
--- a/layers/park/park.sql
+++ b/layers/park/park.sql
@@ -26,7 +26,7 @@ SELECT osm_id,
FROM (
SELECT osm_id,
geometry,
- park_class(boundary, leisure, landuse, historic) AS class,
+ park_class(boundary, leisure, landuse, historic, seamark_type) AS class,
name,
name_en,
name_de,
@@ -42,7 +42,7 @@ FROM (
NULL AS tags,
NULL AS leisure,
NULL AS landuse,
- NULL AS boundary,
+ CASE WHEN boundary='aboriginal_lands' THEN boundary END AS boundary,
NULL AS historic
FROM osm_park_polygon_dissolve_z4
WHERE zoom_level = 4
@@ -208,7 +208,7 @@ FROM (
name_de,
tags,
row_number() OVER (
- PARTITION BY LabelGrid(geometry_point, 100 * pixel_width)
+ PARTITION BY LabelGrid(geometry_point, 100 * pixel_width), boundary
ORDER BY
(CASE WHEN boundary = 'national_park' THEN TRUE ELSE FALSE END) DESC,
(COALESCE(NULLIF(tags->'wikipedia', ''), NULLIF(tags->'wikidata', '')) IS NOT NULL) DESC,
@@ -226,6 +226,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z5
WHERE zoom_level = 5
@@ -244,6 +245,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z6
WHERE zoom_level = 6
@@ -262,6 +264,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z7
WHERE zoom_level = 7
@@ -280,6 +283,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z8
WHERE zoom_level = 8
@@ -298,6 +302,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z9
WHERE zoom_level = 9
@@ -316,6 +321,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z10
WHERE zoom_level = 10
@@ -334,6 +340,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z11
WHERE zoom_level = 11
@@ -352,6 +359,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z12
WHERE zoom_level = 12
@@ -370,6 +378,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon_gen_z13
WHERE zoom_level = 13
@@ -388,6 +397,7 @@ FROM (
landuse,
boundary,
historic,
+ seamark_type,
area
FROM osm_park_polygon
WHERE zoom_level >= 14
diff --git a/layers/park/park.yaml b/layers/park/park.yaml
index bc1ade2a..dfffad57 100644
--- a/layers/park/park.yaml
+++ b/layers/park/park.yaml
@@ -1,17 +1,22 @@
layer:
id: "park"
description: |
- The park layer contains parks from OpenStreetMap tagged with
+ The park layer contains large area features from OpenStreetMap tagged with
[`boundary=national_park`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dnational_park),
[`boundary=protected_area`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dprotected_area),
[`leisure=nature_reserve`](http://wiki.openstreetmap.org/wiki/Tag:leisure%3Dnature_reserve),
or select values of [`historic=*`](http://wiki.openstreetmap.org/wiki/Key:historic).
+ This layer also includes boundaries for indigenous lands tagged with
+ [`boundary=aboriginal_lands`](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Daboriginal_lands).
+ Indigenous boundaries are not parks, but they are included in this layer for technical reasons related to data processing.
+ These boundaries represent areas with special legal and administrative status for indigenous peoples.
buffer_size: 4
fields:
class:
description: |
- Use the **class** to differentiate between different parks.
+ Use the **class** to differentiate between different large areas.
The class for `boundary=protected_area` parks is one of: `national_park`, `nature_reserve`, `recreation_ground`, or `historic`.
+ The class for `boundary=aboriginal_lands` is `aboriginal_lands`.
name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the park (point features only).
name_en: English name `name:en` if available, otherwise `name` (point features only).
name_de: German name `name:de` if available, otherwise `name` or `name:en` (point features only).
diff --git a/layers/park/update_park_polygon.sql b/layers/park/update_park_polygon.sql
index 69492692..5c32b2e5 100644
--- a/layers/park/update_park_polygon.sql
+++ b/layers/park/update_park_polygon.sql
@@ -24,14 +24,16 @@ DROP MATERIALIZED VIEW IF EXISTS osm_park_polygon_dissolve_z4 CASCADE;
CREATE MATERIALIZED VIEW osm_park_polygon_dissolve_z4 AS
(
SELECT min(osm_id) AS osm_id,
- ST_Union(geometry) AS geometry
+ ST_Union(geometry) AS geometry,
+ boundary
FROM (
SELECT ST_ClusterDBSCAN(geometry, 0, 1) OVER() AS cluster,
osm_id,
- geometry
+ geometry,
+ boundary
FROM osm_park_polygon_gen_z4
) park_cluster
- GROUP BY cluster
+ GROUP BY boundary, cluster
);
CREATE UNIQUE INDEX IF NOT EXISTS osm_park_polygon_dissolve_idx ON osm_park_polygon_dissolve_z4 (osm_id);
@@ -46,6 +48,8 @@ DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z7;
DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z6;
DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z5;
DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z4;
+DROP TRIGGER IF EXISTS trigger_flag ON osm_park_polygon;
+DROP TRIGGER IF EXISTS trigger_refresh ON park_polygon.updates;
-- etldoc: osm_park_polygon -> osm_park_polygon
-- etldoc: osm_park_polygon_gen_z13 -> osm_park_polygon_gen_z13
@@ -119,6 +123,42 @@ CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z5_point_geom_idx ON osm_park_po
CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z4_polygon_geom_idx ON osm_park_polygon_gen_z4 USING gist (geometry);
CREATE INDEX IF NOT EXISTS osm_park_polygon_dissolve_z4_polygon_geom_idx ON osm_park_polygon_dissolve_z4 USING gist (geometry);
+CREATE SCHEMA IF NOT EXISTS park_polygon;
+
+CREATE TABLE IF NOT EXISTS park_polygon.updates
+(
+ id serial PRIMARY KEY,
+ t text,
+ UNIQUE (t)
+);
+
+CREATE OR REPLACE FUNCTION park_polygon.flag() RETURNS trigger AS
+$$
+BEGIN
+ INSERT INTO park_polygon.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION park_polygon.refresh() RETURNS trigger AS
+$$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh park_polygon';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE osm_park_polygon_gen_z4;
+ REFRESH MATERIALIZED VIEW osm_park_polygon_dissolve_z4;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM park_polygon.updates;
+
+ RAISE LOG 'Refresh park_polygon done in %', age(clock_timestamp(), t);
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
CREATE OR REPLACE FUNCTION update_osm_park_polygon_row()
RETURNS trigger
AS
@@ -206,3 +246,15 @@ CREATE TRIGGER update_row
FOR EACH ROW
EXECUTE PROCEDURE update_osm_park_dissolved_polygon_row();
+CREATE TRIGGER trigger_flag
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_park_polygon
+ FOR EACH STATEMENT
+EXECUTE PROCEDURE park_polygon.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+ AFTER INSERT
+ ON park_polygon.updates
+ INITIALLY DEFERRED
+ FOR EACH ROW
+EXECUTE PROCEDURE park_polygon.refresh();
diff --git a/layers/place/update_city_point.sql b/layers/place/update_city_point.sql
index 721bcd70..ef831127 100644
--- a/layers/place/update_city_point.sql
+++ b/layers/place/update_city_point.sql
@@ -8,7 +8,7 @@ CREATE SCHEMA IF NOT EXISTS place_city;
CREATE TABLE IF NOT EXISTS place_city.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
CREATE OR REPLACE FUNCTION update_osm_city_point(full_update boolean) RETURNS void AS
@@ -49,18 +49,12 @@ $$ LANGUAGE SQL;
SELECT update_osm_city_point(true);
-CREATE INDEX IF NOT EXISTS osm_city_point_rank_idx ON osm_city_point ("rank");
-
-- Handle updates
CREATE OR REPLACE FUNCTION place_city.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_city.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_city.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_city.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -85,6 +79,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_city rank';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_city.osm_ids;
+ ANALYZE osm_city_point;
+
PERFORM update_osm_city_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_city.osm_ids;
@@ -97,13 +96,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_city_point
FOR EACH ROW
EXECUTE PROCEDURE place_city.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_city_point
FOR EACH STATEMENT
EXECUTE PROCEDURE place_city.flag();
diff --git a/layers/place/update_continent_point.sql b/layers/place/update_continent_point.sql
index 5e944a2a..3d920635 100644
--- a/layers/place/update_continent_point.sql
+++ b/layers/place/update_continent_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS place_continent_point;
CREATE TABLE IF NOT EXISTS place_continent_point.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_continent_point -> osm_continent_point
@@ -26,11 +26,7 @@ SELECT update_osm_continent_point(true);
CREATE OR REPLACE FUNCTION place_continent_point.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_continent_point.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_continent_point.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_continent_point.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -55,6 +51,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_continent_point';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_continent_point.osm_ids;
+ ANALYZE osm_continent_point;
+
PERFORM update_osm_continent_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_continent_point.osm_ids;
@@ -67,13 +68,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_continent_point
FOR EACH ROW
EXECUTE PROCEDURE place_continent_point.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_continent_point
FOR EACH STATEMENT
EXECUTE PROCEDURE place_continent_point.flag();
diff --git a/layers/place/update_country_point.sql b/layers/place/update_country_point.sql
index a4cc79df..adc8004e 100644
--- a/layers/place/update_country_point.sql
+++ b/layers/place/update_country_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS place_country;
CREATE TABLE IF NOT EXISTS place_country.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: ne_10m_admin_0_countries -> osm_country_point
@@ -98,18 +98,12 @@ $$ LANGUAGE SQL;
SELECT update_osm_country_point(true);
-CREATE INDEX IF NOT EXISTS osm_country_point_rank_idx ON osm_country_point ("rank");
-
-- Handle updates
CREATE OR REPLACE FUNCTION place_country.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_country.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_country.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_country.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -134,6 +128,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_country rank';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_country.osm_ids;
+ ANALYZE osm_country_point;
+
PERFORM update_osm_country_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_country.osm_ids;
@@ -146,13 +145,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_country_point
FOR EACH ROW
EXECUTE PROCEDURE place_country.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_country_point
FOR EACH STATEMENT
EXECUTE PROCEDURE place_country.flag();
diff --git a/layers/place/update_island_point.sql b/layers/place/update_island_point.sql
index 7f57d323..3503464a 100644
--- a/layers/place/update_island_point.sql
+++ b/layers/place/update_island_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS place_island_point;
CREATE TABLE IF NOT EXISTS place_island_point.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_island_point -> osm_island_point
@@ -26,11 +26,7 @@ SELECT update_osm_island_point(true);
CREATE OR REPLACE FUNCTION place_island_point.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_island_point.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_island_point.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_island_point.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -55,6 +51,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_island_point';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_island_point.osm_ids;
+ ANALYZE osm_island_point;
+
PERFORM update_osm_island_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_island_point.osm_ids;
@@ -67,13 +68,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_island_point
FOR EACH ROW
EXECUTE PROCEDURE place_island_point.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_island_point
FOR EACH STATEMENT
EXECUTE PROCEDURE place_island_point.flag();
diff --git a/layers/place/update_island_polygon.sql b/layers/place/update_island_polygon.sql
index 2644f355..23e76c7e 100644
--- a/layers/place/update_island_polygon.sql
+++ b/layers/place/update_island_polygon.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS place_island_polygon;
CREATE TABLE IF NOT EXISTS place_island_polygon.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_island_polygon -> osm_island_polygon
@@ -33,11 +33,7 @@ SELECT update_osm_island_polygon(true);
CREATE OR REPLACE FUNCTION place_island_polygon.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_island_polygon.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_island_polygon.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_island_polygon.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -62,6 +58,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_island_polygon';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_island_polygon.osm_ids;
+ ANALYZE osm_island_polygon;
+
PERFORM update_osm_island_polygon(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_island_polygon.osm_ids;
@@ -74,13 +75,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_island_polygon
FOR EACH ROW
EXECUTE PROCEDURE place_island_polygon.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_island_polygon
FOR EACH STATEMENT
EXECUTE PROCEDURE place_island_polygon.flag();
diff --git a/layers/place/update_state_point.sql b/layers/place/update_state_point.sql
index fa155802..9c75c3ed 100644
--- a/layers/place/update_state_point.sql
+++ b/layers/place/update_state_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS place_state;
CREATE TABLE IF NOT EXISTS place_state.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: ne_10m_admin_1_states_provinces -> osm_state_point
@@ -60,18 +60,12 @@ $$ LANGUAGE SQL;
SELECT update_osm_state_point(true);
-CREATE INDEX IF NOT EXISTS osm_state_point_rank_idx ON osm_state_point ("rank");
-
-- Handle updates
CREATE OR REPLACE FUNCTION place_state.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO place_state.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO place_state.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO place_state.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -96,6 +90,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh place_state rank';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE place_state.osm_ids;
+ ANALYZE osm_state_point;
+
PERFORM update_osm_state_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM place_state.osm_ids;
@@ -108,13 +107,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_state_point
FOR EACH ROW
EXECUTE PROCEDURE place_state.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_state_point
FOR EACH STATEMENT
EXECUTE PROCEDURE place_state.flag();
diff --git a/layers/poi/mapping.yaml b/layers/poi/mapping.yaml
index 78166973..117705cc 100644
--- a/layers/poi/mapping.yaml
+++ b/layers/poi/mapping.yaml
@@ -18,6 +18,7 @@ def_poi_mapping_amenity: &poi_mapping_amenity
- biergarten
- bus_station
- cafe
+ - charging_station
- cinema
- clinic
- college
@@ -118,7 +119,66 @@ def_poi_mapping_leisure: &poi_mapping_leisure
# office values , see http://taginfo.openstreetmap.org/keys/office#values
def_poi_mapping_office: &poi_mapping_office
+ - accountant
+ - advertising_agency
+ - architect
+ - association
+ - bail_bond_agent
+ - charity
+ - company
+ - construction_company
+ - consulting
+ - cooperative
+ - courier
+ - coworking
- diplomatic
+ - educational_institution
+ - employment_agency
+ - energy_supplier
+ - engineer
+ - estate_agent
+ - financial
+ - financial_advisor
+ - forestry
+ - foundation
+ - geodesist
+ - government
+ - graphic_design
+ - guide
+ - harbour_master
+ - health_insurance
+ - insurance
+ - interior_design
+ - it
+ - lawyer
+ - logistics
+ - marketing
+ - moving_company
+ - newspaper
+ - ngo
+ - notary
+ - physician
+ - political_party
+ - private_investigator
+ - property_management
+ - publisher
+ - quango
+ - religion
+ - research
+ - security
+ - surveyor
+ - tax_advisor
+ - taxi
+ - telecommunication
+ - therapist
+ - translator
+ - travel_agent
+ - tutoring
+ - union
+ - university
+ - water_utility
+ - web_design
+ - wedding_planner
# railway values , see http://taginfo.openstreetmap.org/keys/railway#values
def_poi_mapping_railway: &poi_mapping_railway
@@ -193,6 +253,7 @@ def_poi_mapping_shop: &poi_mapping_shop
- newsagent
- optician
- outdoor
+ - paint
- perfume
- perfumery
- pet
diff --git a/layers/poi/mapping_diagram.png b/layers/poi/mapping_diagram.png
index fcfbf433..1b5726fc 100644
Binary files a/layers/poi/mapping_diagram.png and b/layers/poi/mapping_diagram.png differ
diff --git a/layers/poi/poi.yaml b/layers/poi/poi.yaml
index 0181ab6c..937292a8 100644
--- a/layers/poi/poi.yaml
+++ b/layers/poi/poi.yaml
@@ -21,10 +21,20 @@ layer:
'coffee', 'computer', 'convenience', 'copyshop', 'cosmetics', 'garden_centre', 'doityourself',
'erotic', 'electronics', 'fabric', 'florist', 'frozen_food', 'furniture', 'video_games', 'video',
'general', 'gift', 'hardware', 'hearing_aids', 'hifi', 'ice_cream', 'interior_decoration',
- 'jewelry', 'kiosk', 'locksmith', 'lamps', 'mall', 'massage', 'motorcycle', 'mobile_phone',
- 'newsagent', 'optician', 'outdoor', 'perfumery', 'perfume', 'pet', 'photo', 'second_hand',
- 'shoes', 'sports', 'stationery', 'tailor', 'tattoo', 'ticket', 'tobacco', 'toys', 'travel_agency',
+ 'jewelry', 'kiosk', 'locksmith', 'lamps', 'mall', 'massage', 'motorcycle', 'mobile_phone',
+ 'newsagent', 'optician', 'outdoor', 'paint', 'perfumery', 'perfume', 'pet', 'photo', 'second_hand',
+ 'shoes', 'sports', 'stationery', 'tailor', 'tattoo', 'ticket', 'tobacco', 'toys', 'travel_agency',
'watches', 'weapons', 'wholesale']
+ office:
+ subclass: ['accountant', 'advertising_agency', 'architect', 'association', 'bail_bond_agent', 'charity',
+ 'company', 'construction_company', 'consulting', 'cooperative', 'courier', 'coworking', 'diplomatic',
+ 'educational_institution', 'employment_agency', 'energy_supplier', 'engineer', 'estate_agent', 'financial',
+ 'financial_advisor', 'forestry', 'foundation', 'geodesist', 'government', 'graphic_design', 'guide',
+ 'harbour_master', 'health_insurance', 'insurance', 'interior_design', 'it', 'lawyer', 'logistics', 'marketing',
+ 'moving_company', 'newspaper', 'ngo', 'notary', 'physician', 'political_party', 'private_investigator',
+ 'property_management', 'publisher', 'quango', 'religion', 'research', 'security', 'surveyor', 'tax_advisor',
+ 'taxi', 'telecommunication', 'therapist', 'translator', 'travel_agent', 'tutoring', 'union', 'university',
+ 'water_utility', 'web_design', 'wedding_planner']
town_hall:
subclass: ['townhall', 'public_building', 'courthouse', 'community_centre']
golf:
@@ -96,6 +106,8 @@ layer:
subclass: ['castle', 'ruins']
atm:
subclass: ['atm']
+ fuel:
+ subclass: ['fuel', 'charging_station']
subclass:
description: |
Original value of either the
diff --git a/layers/poi/style.json b/layers/poi/style.json
index 3c5fa70b..33c08857 100644
--- a/layers/poi/style.json
+++ b/layers/poi/style.json
@@ -743,7 +743,27 @@
]
]
},
- "icon-image": "{subclass}",
+ "icon-image": [
+ "coalesce",
+ [
+ "image",
+ [
+ "get",
+ "subclass"
+ ]
+ ],
+ [
+ "image",
+ [
+ "get",
+ "class"
+ ]
+ ],
+ [
+ "image",
+ ""
+ ]
+ ],
"text-field": "{name:latin}\n{name:nonlatin}",
"visibility": "visible",
"text-anchor": "top",
@@ -778,7 +798,8 @@
"information",
"cinema",
"theatre",
- "fire_station"
+ "fire_station",
+ "office"
],
[
"!=",
@@ -976,7 +997,8 @@
"class",
"car",
"bicycle_parking",
- "fuel"
+ "fuel",
+ "charging_station"
]
],
"order": 170
diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql
index 2218543e..ddabb779 100644
--- a/layers/poi/update_poi_point.sql
+++ b/layers/poi/update_poi_point.sql
@@ -1,18 +1,28 @@
DROP TRIGGER IF EXISTS trigger_flag ON osm_poi_point;
DROP TRIGGER IF EXISTS trigger_refresh ON poi_point.updates;
+DROP TRIGGER IF EXISTS trigger_store ON osm_poi_point;
+
+CREATE SCHEMA IF NOT EXISTS poi_point;
+
+CREATE TABLE IF NOT EXISTS poi_point.osm_ids
+(
+ osm_id bigint PRIMARY KEY
+);
-- etldoc: osm_poi_point -> osm_poi_point
-CREATE OR REPLACE FUNCTION update_osm_poi_point() RETURNS void AS
+CREATE OR REPLACE FUNCTION update_osm_poi_point(full_update bool) RETURNS void AS
$$
BEGIN
UPDATE osm_poi_point
SET subclass = 'subway'
- WHERE station = 'subway'
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_point.osm_ids))
+ AND station = 'subway'
AND subclass = 'station';
UPDATE osm_poi_point
SET subclass = 'halt'
- WHERE funicular = 'yes'
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_point.osm_ids))
+ AND funicular = 'yes'
AND subclass = 'station';
-- ATM without name
@@ -23,7 +33,8 @@ BEGIN
COALESCE(tags -> 'operator', tags -> 'network'),
tags || hstore('name', COALESCE(tags -> 'operator', tags -> 'network'))
)
- WHERE subclass = 'atm'
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_point.osm_ids))
+ AND subclass = 'atm'
AND name = ''
AND COALESCE(tags -> 'operator', tags -> 'network') IS NOT NULL;
@@ -32,22 +43,24 @@ BEGIN
-- (using name for parcel lockers is discouraged, see osm wiki)
UPDATE osm_poi_point
SET (name, tags) = (
- CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref')),
- tags || hstore('name', CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref')))
+ TRIM(CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref'))),
+ tags || hstore('name', TRIM(CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref'))))
)
- WHERE subclass = 'parcel_locker'
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_point.osm_ids))
+ AND subclass IN ('parcel_locker', 'charging_station')
AND name = ''
AND COALESCE(tags -> 'brand', tags -> 'operator') IS NOT NULL;
UPDATE osm_poi_point
SET tags = update_tags(tags, geometry)
- WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_point.osm_ids))
+ AND COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL
AND tags != update_tags(tags, geometry);
END;
$$ LANGUAGE plpgsql;
-SELECT update_osm_poi_point();
+SELECT update_osm_poi_point(TRUE);
-- etldoc: osm_poi_stop_rank -> osm_poi_point
CREATE OR REPLACE FUNCTION update_osm_poi_point_agg() RETURNS void AS
@@ -91,7 +104,13 @@ SELECT update_osm_poi_point_agg();
-- Handle updates
-CREATE SCHEMA IF NOT EXISTS poi_point;
+CREATE OR REPLACE FUNCTION poi_point.store() RETURNS trigger AS
+$$
+BEGIN
+ INSERT INTO poi_point.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
CREATE TABLE IF NOT EXISTS poi_point.updates
(
@@ -113,11 +132,18 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh poi_point';
- PERFORM update_osm_poi_point();
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE poi_point.osm_ids;
+ ANALYZE osm_poi_point;
+
+ PERFORM update_osm_poi_point(FALSE);
REFRESH MATERIALIZED VIEW osm_poi_stop_centroid;
REFRESH MATERIALIZED VIEW osm_poi_stop_rank;
PERFORM update_osm_poi_point_agg();
-- noinspection SqlWithoutWhere
+ DELETE FROM poi_point.osm_ids;
+ -- noinspection SqlWithoutWhere
DELETE FROM poi_point.updates;
RAISE LOG 'Refresh poi_point done in %', age(clock_timestamp(), t);
@@ -125,8 +151,14 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
+CREATE TRIGGER trigger_store
+ AFTER INSERT OR UPDATE
+ ON osm_poi_point
+ FOR EACH ROW
+EXECUTE PROCEDURE poi_point.store();
+
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_poi_point
FOR EACH STATEMENT
EXECUTE PROCEDURE poi_point.flag();
diff --git a/layers/poi/update_poi_polygon.sql b/layers/poi/update_poi_polygon.sql
index cf6c2238..fc8db960 100644
--- a/layers/poi/update_poi_polygon.sql
+++ b/layers/poi/update_poi_polygon.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS poi_polygon;
CREATE TABLE IF NOT EXISTS poi_polygon.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
-- etldoc: osm_poi_polygon -> osm_poi_polygon
@@ -36,6 +36,19 @@ $$
AND funicular = 'yes'
AND subclass = 'station';
+ -- Parcel locker and charging_station without name
+ -- use either brand or operator and add ref if present
+ -- (using name for parcel lockers is discouraged, see osm wiki)
+ UPDATE osm_poi_polygon
+ SET (name, tags) = (
+ TRIM(CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref'))),
+ tags || hstore('name', TRIM(CONCAT(COALESCE(tags -> 'brand', tags -> 'operator'), concat(' ', tags -> 'ref'))))
+ )
+ WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_polygon.osm_ids))
+ AND subclass IN ('parcel_locker', 'charging_station')
+ AND name = ''
+ AND COALESCE(tags -> 'brand', tags -> 'operator') IS NOT NULL;
+
UPDATE osm_poi_polygon
SET tags = update_tags(tags, geometry)
WHERE (full_update OR osm_id IN (SELECT osm_id FROM poi_polygon.osm_ids))
@@ -51,11 +64,7 @@ SELECT update_poi_polygon(true);
CREATE OR REPLACE FUNCTION poi_polygon.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO poi_polygon.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO poi_polygon.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO poi_polygon.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -80,6 +89,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh poi_polygon';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE poi_polygon.osm_ids;
+ ANALYZE osm_poi_polygon;
+
PERFORM update_poi_polygon(false);
-- noinspection SqlWithoutWhere
DELETE FROM poi_polygon.osm_ids;
@@ -92,13 +106,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_poi_polygon
FOR EACH ROW
EXECUTE PROCEDURE poi_polygon.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_poi_polygon
FOR EACH STATEMENT
EXECUTE PROCEDURE poi_polygon.flag();
diff --git a/layers/transportation/etl_diagram.png b/layers/transportation/etl_diagram.png
index 6c3eb0d3..b21fb86d 100644
Binary files a/layers/transportation/etl_diagram.png and b/layers/transportation/etl_diagram.png differ
diff --git a/layers/transportation/mapping.yaml b/layers/transportation/mapping.yaml
index 1f01322b..ff1b6b87 100644
--- a/layers/transportation/mapping.yaml
+++ b/layers/transportation/mapping.yaml
@@ -31,6 +31,48 @@ generalized_tables:
sql_filter: ST_IsValid(geometry)
tolerance: ZRES13
+# etldoc: osm_shipway_linestring_gen_z5 -> osm_shipway_linestring_gen_z4
+ shipway_linestring_gen_z4:
+ source: shipway_linestring_gen_z5
+ sql_filter: ST_Length(geometry)>2*ZRES0
+ tolerance: ZRES5
+
+# etldoc: osm_shipway_linestring_gen_z6 -> osm_shipway_linestring_gen_z5
+ shipway_linestring_gen_z5:
+ source: shipway_linestring_gen_z6
+ sql_filter: ST_Length(geometry)>ZRES0
+ tolerance: ZRES6
+
+# etldoc: osm_shipway_linestring_gen_z7 -> osm_shipway_linestring_gen_z6
+ shipway_linestring_gen_z6:
+ source: shipway_linestring_gen_z7
+ sql_filter: ST_Length(geometry)>ZRES1
+ tolerance: ZRES7
+
+# etldoc: osm_shipway_linestring_gen_z8 -> osm_shipway_linestring_gen_z7
+ shipway_linestring_gen_z7:
+ source: shipway_linestring_gen_z8
+ sql_filter: ST_Length(geometry)>ZRES2
+ tolerance: ZRES8
+
+# etldoc: osm_shipway_linestring_gen_z9 -> osm_shipway_linestring_gen_z8
+ shipway_linestring_gen_z8:
+ source: shipway_linestring_gen_z9
+ sql_filter: ST_Length(geometry)>ZRES3
+ tolerance: ZRES9
+
+# etldoc: osm_shipway_linestring_gen_z10 -> osm_shipway_linestring_gen_z9
+ shipway_linestring_gen_z9:
+ source: shipway_linestring_gen_z10
+ sql_filter: ST_Length(geometry)>ZRES4
+ tolerance: ZRES10
+
+# etldoc: osm_shipway_linestring_gen_z11 -> osm_shipway_linestring_gen_z10
+ shipway_linestring_gen_z10:
+ source: shipway_linestring_gen_z11
+ sql_filter: ST_Length(geometry)>ZRES5
+ tolerance: ZRES11
+
# etldoc: osm_shipway_linestring_gen_z12 -> osm_shipway_linestring_gen_z11
shipway_linestring_gen_z11:
source: shipway_linestring_gen_z12
@@ -279,7 +321,6 @@ tables:
- *bridge
- *ramp
- *ford
- - *oneway
- *area
- *service
- *usage
@@ -357,7 +398,6 @@ tables:
- *bridge
- *ramp
- *ford
- - *oneway
- *area
- *service
- *usage
diff --git a/layers/transportation/network_type.sql b/layers/transportation/network_type.sql
index 3c621de9..f1fa8640 100644
--- a/layers/transportation/network_type.sql
+++ b/layers/transportation/network_type.sql
@@ -32,15 +32,3 @@ $$
'us-interstate');
$$ LANGUAGE sql IMMUTABLE
PARALLEL SAFE;
-
-DO
-$$
- BEGIN
- BEGIN
- ALTER TABLE osm_route_member
- ADD COLUMN network_type route_network_type;
- EXCEPTION
- WHEN duplicate_column THEN RAISE NOTICE 'column network_type already exists in network_type.';
- END;
- END;
-$$;
diff --git a/layers/transportation/style.json b/layers/transportation/style.json
index 1204bc36..92dddfa8 100644
--- a/layers/transportation/style.json
+++ b/layers/transportation/style.json
@@ -5,13 +5,13 @@
"type": "line",
"source": "openmaptiles",
"source-layer": "transportation",
- "minzoom": 10,
+ "minzoom": 4,
"layout": {
"line-join": "round",
"visibility": "visible"
},
"paint": {
- "line-color": "#7d7df8",
+ "line-color": "#66f",
"line-width": {
"stops": [
[
@@ -25,8 +25,8 @@
]
},
"line-dasharray": [
- 2,
- 2
+ 6,
+ 6
]
},
"filter": [
@@ -5607,7 +5607,7 @@
"stops": [
[
8,
- " #787878"
+ "#787878"
],
[
14,
diff --git a/layers/transportation/transportation.sql b/layers/transportation/transportation.sql
index 7a516f65..85d25220 100644
--- a/layers/transportation/transportation.sql
+++ b/layers/transportation/transportation.sql
@@ -483,7 +483,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -518,7 +518,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -553,7 +553,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -589,7 +589,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -625,7 +625,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -659,7 +659,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -674,6 +674,237 @@ FROM (
WHERE zoom_level >= 13
UNION ALL
+ -- etldoc: osm_shipway_linestring_gen_z4 -> layer_transportation:z4
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z4
+ WHERE zoom_level = 4
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z5 -> layer_transportation:z5
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z5
+ WHERE zoom_level = 5
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z6 -> layer_transportation:z6
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z6
+ WHERE zoom_level = 6
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z7 -> layer_transportation:z7
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z7
+ WHERE zoom_level = 7
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z8 -> layer_transportation:z8
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z8
+ WHERE zoom_level = 8
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z9 -> layer_transportation:z9
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z9
+ WHERE zoom_level = 9
+ UNION ALL
+
+ -- etldoc: osm_shipway_linestring_gen_z10 -> layer_transportation:z10
+ SELECT osm_id,
+ geometry,
+ NULL AS highway,
+ NULL AS construction,
+ NULL AS network,
+ NULL AS railway,
+ NULL AS aerialway,
+ shipway,
+ NULL AS public_transport,
+ service_value(service) AS service,
+ NULL::text AS access,
+ NULL::boolean AS toll,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ NULL::boolean AS expressway,
+ is_ramp,
+ NULL::int AS is_oneway,
+ NULL AS man_made,
+ layer,
+ NULL::int AS level,
+ NULL::boolean AS indoor,
+ NULL AS bicycle,
+ NULL AS foot,
+ NULL AS horse,
+ NULL AS mtb_scale,
+ NULL AS surface,
+ z_order
+ FROM osm_shipway_linestring_gen_z10
+ WHERE zoom_level = 10
+ UNION ALL
+
-- etldoc: osm_shipway_linestring_gen_z11 -> layer_transportation:z11
SELECT osm_id,
geometry,
@@ -692,7 +923,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -725,7 +956,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
@@ -759,7 +990,7 @@ FROM (
is_ford,
NULL::boolean AS expressway,
is_ramp,
- is_oneway,
+ NULL::int AS is_oneway,
NULL AS man_made,
layer,
NULL::int AS level,
diff --git a/layers/transportation/update_route_member.sql b/layers/transportation/update_route_member.sql
index 83361abc..88f7cefe 100644
--- a/layers/transportation/update_route_member.sql
+++ b/layers/transportation/update_route_member.sql
@@ -1,3 +1,5 @@
+DROP TRIGGER IF EXISTS trigger_store_transportation_highway_linestring ON osm_highway_linestring;
+
-- Create bounding windows for country-specific processing
-- etldoc: ne_10m_admin_0_countries -> ne_10m_admin_0_gb_buffer
@@ -16,12 +18,11 @@ WHERE iso_a2 = 'IE';
-- etldoc: osm_highway_linestring -> gbr_route_members_view
-- etldoc: ne_10m_admin_0_gb_buffer -> gbr_route_members_view
CREATE OR REPLACE VIEW gbr_route_members_view AS
-SELECT 0,
- osm_id,
- substring(ref FROM E'^[ABM][0-9ABM()]+'),
+SELECT osm_id AS member,
+ substring(ref FROM E'^[ABM][0-9ABM()]+') AS ref,
-- See https://wiki.openstreetmap.org/wiki/Roads_in_the_United_Kingdom
CASE WHEN highway = 'motorway' THEN 'omt-gb-motorway'
- WHEN highway = 'trunk' THEN 'omt-gb-trunk'
+ WHEN highway = 'trunk' THEN 'omt-gb-trunk'
WHEN highway IN ('primary','secondary') THEN 'omt-gb-primary' END AS network
FROM osm_highway_linestring
WHERE length(ref) > 1
@@ -32,12 +33,11 @@ WHERE length(ref) > 1
-- etldoc: osm_highway_linestring -> ire_route_members_view
-- etldoc: ne_10m_admin_0_ie_buffer -> ire_route_members_view
CREATE OR REPLACE VIEW ire_route_members_view AS
-SELECT 0,
- osm_id,
- substring(ref FROM E'^[MNRL][0-9]+'),
+SELECT osm_id AS member,
+ substring(ref FROM E'^[MNRL][0-9]+') AS ref,
-- See https://wiki.openstreetmap.org/wiki/Ireland/Roads
CASE WHEN highway = 'motorway' THEN 'omt-ie-motorway'
- WHEN highway IN ('trunk','primary') THEN 'omt-ie-national'
+ WHEN highway IN ('trunk','primary') THEN 'omt-ie-national'
ELSE 'omt-ie-regional' END AS network
FROM osm_highway_linestring
WHERE length(ref) > 1
@@ -45,23 +45,6 @@ WHERE length(ref) > 1
AND highway IN ('motorway', 'trunk', 'primary', 'secondary', 'unclassified')
;
--- Create GBR/IRE relations (so we can use it in the same way as other relations)
--- etldoc: osm_route_member -> osm_route_member
-DELETE
-FROM osm_route_member
-WHERE network IN ('omt-gb-motorway', 'omt-gb-trunk', 'omt-gb-primary',
- 'omt-ie-motorway', 'omt-ie-national', 'omt-ie-national');
-
--- etldoc: gbr_route_members_view -> osm_route_member
-INSERT INTO osm_route_member (osm_id, member, ref, network)
-SELECT *
-FROM gbr_route_members_view;
-
--- etldoc: ire_route_members_view -> osm_route_member
-INSERT INTO osm_route_member (osm_id, member, ref, network)
-SELECT *
-FROM ire_route_members_view;
-
CREATE OR REPLACE FUNCTION osm_route_member_network_type(network text, ref text) RETURNS route_network_type AS
$$
SELECT CASE
@@ -75,7 +58,7 @@ SELECT CASE
CASE
WHEN ref LIKE '4__' THEN 'ca-provincial-arterial'::route_network_type
WHEN ref = 'QEW' THEN 'ca-provincial-arterial'::route_network_type
- ELSE 'ca-provincial-arterial'::route_network_type
+ ELSE 'ca-provincial'::route_network_type
END
WHEN network = 'CA:MB:PTH' AND ref = '75' THEN 'ca-provincial-arterial'::route_network_type
WHEN network = 'CA:AB:primary' AND ref IN ('2','3','4') THEN 'ca-provincial-arterial'::route_network_type
@@ -91,104 +74,145 @@ SELECT CASE
$$ LANGUAGE sql IMMUTABLE
PARALLEL SAFE;
--- etldoc: osm_route_member -> osm_route_member
--- see http://wiki.openstreetmap.org/wiki/Relation:route#Road_routes
-UPDATE osm_route_member
-SET network_type = osm_route_member_network_type(network, ref)
-WHERE network != ''
- AND network_type IS DISTINCT FROM osm_route_member_network_type(network, ref)
-;
+CREATE TABLE IF NOT EXISTS transportation_route_member_coalesced
+(
+ member bigint,
+ network varchar,
+ ref varchar,
+ osm_id bigint not null,
+ role varchar,
+ type smallint,
+ name varchar,
+ osmc_symbol varchar,
+ colour varchar,
+ network_type route_network_type,
+ concurrency_index integer,
+ rank integer,
+ PRIMARY KEY (member, network, ref)
+);
-CREATE OR REPLACE FUNCTION update_osm_route_member() RETURNS void AS
+CREATE OR REPLACE FUNCTION update_osm_route_member(full_update bool) RETURNS void AS
$$
BEGIN
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation_name.network_changes;
+ ANALYZE osm_highway_linestring;
+ ANALYZE osm_route_member;
+
DELETE
- FROM osm_route_member AS r
- USING
- transportation_name.network_changes AS c
- WHERE network IN ('omt-gb-motorway', 'omt-gb-trunk', 'omt-gb-primary',
- 'omt-ie-motorway', 'omt-ie-national', 'omt-ie-regional')
- AND r.osm_id = c.osm_id;
+ FROM transportation_route_member_coalesced
+ USING transportation_name.network_changes c
+ WHERE c.is_old IS TRUE AND transportation_route_member_coalesced.member = c.osm_id;
- INSERT INTO osm_route_member (osm_id, member, ref, network)
- SELECT r.*
- FROM gbr_route_members_view AS r
- JOIN transportation_name.network_changes AS c ON
- r.osm_id = c.osm_id;
+ -- Create GBR/IRE relations (so we can use it in the same way as other relations)
+ -- etldoc: gbr_route_members_view -> transportation_route_member_coalesced
+ INSERT INTO transportation_route_member_coalesced (member, network, ref, network_type, concurrency_index, osm_id)
+ SELECT member, network, coalesce(ref, '') AS ref,
+ osm_route_member_network_type(network, coalesce(ref, '')) AS network_type,
+ 1 AS concurrency_index, 0 AS osm_id
+ FROM gbr_route_members_view
+ WHERE full_update OR EXISTS(
+ SELECT NULL
+ FROM transportation_name.network_changes c
+ WHERE c.is_old IS FALSE AND c.osm_id = gbr_route_members_view.member
+ )
+ GROUP BY member, network, coalesce(ref, '')
+ ON CONFLICT (member, network, ref) DO NOTHING;
- INSERT INTO osm_route_member (osm_id, member, ref, network)
- SELECT r.*
- FROM ire_route_members_view AS r
- JOIN transportation_name.network_changes AS c ON
- r.osm_id = c.osm_id;
+ -- etldoc: ire_route_members_view -> transportation_route_member_coalesced
+ INSERT INTO transportation_route_member_coalesced (member, network, ref, network_type, concurrency_index, osm_id)
+ SELECT member, network, coalesce(ref, '') AS ref,
+ osm_route_member_network_type(network, coalesce(ref, '')) AS network_type,
+ 1 AS concurrency_index, 0 AS osm_id
+ FROM ire_route_members_view
+ WHERE full_update OR EXISTS(
+ SELECT NULL
+ FROM transportation_name.network_changes c
+ WHERE c.is_old IS FALSE AND c.osm_id = ire_route_members_view.member
+ )
+ GROUP BY member, network, coalesce(ref, '')
+ ON CONFLICT (member, network, ref) DO NOTHING;
- INSERT INTO osm_route_member (id, osm_id, network_type, concurrency_index, rank)
+ -- etldoc: osm_route_member -> transportation_route_member_coalesced
+ INSERT INTO transportation_route_member_coalesced
SELECT
- id,
- osm_id,
+ osm_route_member_filtered.*,
osm_route_member_network_type(network, ref) AS network_type,
- DENSE_RANK() over (PARTITION BY member ORDER BY network_type, network, LENGTH(ref), ref) AS concurrency_index,
+ DENSE_RANK() OVER (
+ PARTITION BY member
+ ORDER BY osm_route_member_network_type(network, ref), network, LENGTH(ref), ref
+ ) AS concurrency_index,
CASE
WHEN network IN ('iwn', 'nwn', 'rwn') THEN 1
WHEN network = 'lwn' THEN 2
WHEN osmc_symbol || colour <> '' THEN 2
END AS rank
- FROM osm_route_member rm
- WHERE rm.member IN
- (SELECT DISTINCT osm_id FROM transportation_name.network_changes)
- ON CONFLICT (id, osm_id) DO UPDATE SET concurrency_index = EXCLUDED.concurrency_index,
- rank = EXCLUDED.rank,
- network_type = EXCLUDED.network_type;
+ FROM (
+ -- etldoc: osm_route_member -> osm_route_member
+ -- see http://wiki.openstreetmap.org/wiki/Relation:route#Road_routes
+ SELECT DISTINCT ON (member, COALESCE(network, ''), COALESCE(ref, ''))
+ member,
+ COALESCE(network, '') AS network,
+ COALESCE(ref, '') AS ref,
+ osm_id,
+ role,
+ type,
+ name,
+ osmc_symbol,
+ colour
+ FROM osm_route_member
+ WHERE full_update OR EXISTS(
+ SELECT NULL
+ FROM transportation_name.network_changes c
+ WHERE c.is_old IS FALSE AND c.osm_id = osm_route_member.member
+ )
+ ) osm_route_member_filtered
+ ON CONFLICT (member, network, ref) DO UPDATE SET osm_id = EXCLUDED.osm_id, role = EXCLUDED.role,
+ type = EXCLUDED.type, name = EXCLUDED.name,
+ osmc_symbol = EXCLUDED.osmc_symbol, colour = EXCLUDED.colour,
+ concurrency_index = EXCLUDED.concurrency_index,
+ rank = EXCLUDED.rank;
END;
$$ LANGUAGE plpgsql;
-CREATE INDEX IF NOT EXISTS osm_route_member_network_idx ON osm_route_member ("network", "ref");
-CREATE INDEX IF NOT EXISTS osm_route_member_member_idx ON osm_route_member ("member");
-CREATE INDEX IF NOT EXISTS osm_route_member_name_idx ON osm_route_member ("name");
-CREATE INDEX IF NOT EXISTS osm_route_member_ref_idx ON osm_route_member ("ref");
+-- Indexes which can be utilized during full-update for queries originating from update_osm_route_member() function
+CREATE INDEX IF NOT EXISTS osm_route_member_member_network_ref_idx
+ ON osm_route_member (member, COALESCE(network, ''), COALESCE(ref, ''));
-CREATE INDEX IF NOT EXISTS osm_route_member_network_type_idx ON osm_route_member ("network_type");
+-- Analyze created index
+ANALYZE osm_route_member;
-/**
-* Discard duplicate routes
-*/
-DELETE FROM osm_route_member WHERE id IN
- (SELECT id
- FROM (SELECT id,
- ROW_NUMBER() OVER (partition BY member, network, ref ORDER BY id) AS rnum
- FROM osm_route_member) t
- WHERE t.rnum > 1);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_route_member_network_ref_idx ON osm_route_member ("member", "network", "ref");
+-- Ensure transportation_name.network_changes table exists since it is required by update_osm_route_member
+CREATE SCHEMA IF NOT EXISTS transportation_name;
+CREATE TABLE IF NOT EXISTS transportation_name.network_changes
+(
+ is_old bool,
+ osm_id bigint,
+ PRIMARY KEY (is_old, osm_id)
+);
-CREATE INDEX IF NOT EXISTS osm_highway_linestring_osm_id_idx ON osm_highway_linestring ("osm_id");
-CREATE UNIQUE INDEX IF NOT EXISTS osm_highway_linestring_gen_z11_osm_id_idx ON osm_highway_linestring_gen_z11 ("osm_id");
+-- Fill transportation_route_member_coalesced table
+TRUNCATE transportation_route_member_coalesced;
+SELECT update_osm_route_member(TRUE);
-ALTER TABLE osm_route_member ADD COLUMN IF NOT EXISTS concurrency_index int,
- ADD COLUMN IF NOT EXISTS rank int;
+-- Index for queries against transportation_route_member_coalesced during transportation-name-network updates
+CREATE INDEX IF NOT EXISTS transportation_route_member_member_idx ON
+ transportation_route_member_coalesced ("member", "concurrency_index");
--- One-time load of concurrency indexes; updates occur via trigger
--- etldoc: osm_route_member -> osm_route_member
-INSERT INTO osm_route_member (id, osm_id, concurrency_index, rank)
- SELECT
- id,
- osm_id,
- DENSE_RANK() over (PARTITION BY member ORDER BY network_type, network, LENGTH(ref), ref) AS concurrency_index,
- CASE
- WHEN network IN ('iwn', 'nwn', 'rwn') THEN 1
- WHEN network = 'lwn' THEN 2
- WHEN osmc_symbol || colour <> '' THEN 2
- END AS rank
- FROM osm_route_member
- ON CONFLICT (id, osm_id) DO UPDATE SET concurrency_index = EXCLUDED.concurrency_index, rank = EXCLUDED.rank;
+-- Analyze populated table with indexes
+ANALYZE transportation_route_member_coalesced;
+
+-- Ensure OSM-ID index exists on osm_highway_linestring
+CREATE UNIQUE INDEX IF NOT EXISTS osm_highway_linestring_osm_id_idx ON osm_highway_linestring ("osm_id");
-- etldoc: osm_route_member -> osm_highway_linestring
UPDATE osm_highway_linestring hl
SET network = rm.network_type
- FROM osm_route_member rm
+ FROM transportation_route_member_coalesced rm
WHERE hl.osm_id=rm.member AND rm.concurrency_index=1;
-- etldoc: osm_route_member -> osm_highway_linestring_gen_z11
UPDATE osm_highway_linestring_gen_z11 hl
SET network = rm.network_type
- FROM osm_route_member rm
+ FROM transportation_route_member_coalesced rm
WHERE hl.osm_id=rm.member AND rm.concurrency_index=1;
diff --git a/layers/transportation/update_transportation_merge.sql b/layers/transportation/update_transportation_merge.sql
index dcf12a50..8756df60 100644
--- a/layers/transportation/update_transportation_merge.sql
+++ b/layers/transportation/update_transportation_merge.sql
@@ -1,11 +1,46 @@
-DROP TRIGGER IF EXISTS trigger_osm_transportation_merge_linestring_gen_z8 ON osm_transportation_merge_linestring_gen_z8;
+DROP TRIGGER IF EXISTS trigger_store_osm_transportation_merge_linestring_gen_z8 ON osm_transportation_merge_linestring_gen_z8;
DROP TRIGGER IF EXISTS trigger_store_transportation_highway_linestring_gen_z9 ON osm_transportation_merge_linestring_gen_z9;
DROP TRIGGER IF EXISTS trigger_flag_transportation_z9 ON osm_transportation_merge_linestring_gen_z9;
DROP TRIGGER IF EXISTS trigger_refresh_z8 ON transportation.updates_z9;
-DROP TRIGGER IF EXISTS trigger_osm_transportation_merge_linestring_gen_z11 ON osm_transportation_merge_linestring_gen_z11;
DROP TRIGGER IF EXISTS trigger_store_transportation_highway_linestring_gen_z11 ON osm_highway_linestring_gen_z11;
+DROP TRIGGER IF EXISTS trigger_store_osm_transportation_merge_linestring_gen_z11 ON osm_transportation_merge_linestring_gen_z11;
DROP TRIGGER IF EXISTS trigger_flag_transportation_z11 ON osm_highway_linestring_gen_z11;
DROP TRIGGER IF EXISTS trigger_refresh_z11 ON transportation.updates_z11;
+DROP TRIGGER IF EXISTS trigger_store_transportation_name_network ON osm_transportation_name_network;
+
+-- Determine whether a segment is long enough to have bridge/tunnel attributes
+-- Dropping small brunnel sections allow for generalization as distinct segments get too small
+CREATE OR REPLACE FUNCTION visible_brunnel(g geometry, brunnel boolean, zoom_level integer)
+ RETURNS boolean AS
+$$
+SELECT
+ brunnel AND
+ -- Width of a tile in meters (111,842 is the length of one degree of latitude at the equator in meters)
+ -- 111,842 * 180 / 2^zoom_level
+ -- = 20131560 / POW(2, zoom_level)
+ -- Drop brunnel if length of way < 2% of tile width (less than 3 pixels)
+ ST_Length(g) *
+ COS(RADIANS(ST_Y(ST_Centroid(ST_Transform(g, 4326))))) *
+ POW(2, zoom_level) / 20131560 > 0.02
+$$ LANGUAGE SQL IMMUTABLE
+ PARALLEL SAFE;
+
+-- Determine whether a segment is long enough to have layer attributes
+CREATE OR REPLACE FUNCTION visible_layer(g geometry, layer int, zoom_level integer)
+ RETURNS int AS
+$$
+SELECT
+ CASE WHEN
+ -- Width of a tile in meters (111,842 is the length of one degree of latitude at the equator in meters)
+ -- 111,842 * 180 / 2^zoom_level
+ -- = 20131560 / POW(2, zoom_level)
+ -- Drop brunnel if length of way < 2% of tile width (less than 3 pixels)
+ ST_Length(g) *
+ COS(RADIANS(ST_Y(ST_Centroid(ST_Transform(g, 4326))))) *
+ POW(2, zoom_level) / 20131560 > 0.02
+ THEN layer END
+$$ LANGUAGE SQL IMMUTABLE
+ PARALLEL SAFE;
-- Instead of using relations to find out the road names we
-- stitch together the touching ways with the same name
@@ -13,7 +48,7 @@ DROP TRIGGER IF EXISTS trigger_refresh_z11 ON transportation.updates_z11;
-- Because this works well for roads that do not have relations as well
-- etldoc: osm_highway_linestring -> osm_transportation_name_network
--- etldoc: osm_route_member -> osm_transportation_name_network
+-- etldoc: transportation_route_member_coalesced -> osm_transportation_name_network
CREATE TABLE IF NOT EXISTS osm_transportation_name_network AS
SELECT
geometry,
@@ -58,30 +93,39 @@ FROM (
hl.z_order,
LEAST(rm1.rank, rm2.rank, rm3.rank, rm4.rank, rm5.rank, rm6.rank) AS route_rank
FROM osm_highway_linestring hl
- LEFT OUTER JOIN osm_route_member rm1 ON rm1.member = hl.osm_id AND rm1.concurrency_index=1
- LEFT OUTER JOIN osm_route_member rm2 ON rm2.member = hl.osm_id AND rm2.concurrency_index=2
- LEFT OUTER JOIN osm_route_member rm3 ON rm3.member = hl.osm_id AND rm3.concurrency_index=3
- LEFT OUTER JOIN osm_route_member rm4 ON rm4.member = hl.osm_id AND rm4.concurrency_index=4
- LEFT OUTER JOIN osm_route_member rm5 ON rm5.member = hl.osm_id AND rm5.concurrency_index=5
- LEFT OUTER JOIN osm_route_member rm6 ON rm6.member = hl.osm_id AND rm6.concurrency_index=6
+ LEFT OUTER JOIN transportation_route_member_coalesced rm1 ON rm1.member = hl.osm_id AND rm1.concurrency_index=1
+ LEFT OUTER JOIN transportation_route_member_coalesced rm2 ON rm2.member = hl.osm_id AND rm2.concurrency_index=2
+ LEFT OUTER JOIN transportation_route_member_coalesced rm3 ON rm3.member = hl.osm_id AND rm3.concurrency_index=3
+ LEFT OUTER JOIN transportation_route_member_coalesced rm4 ON rm4.member = hl.osm_id AND rm4.concurrency_index=4
+ LEFT OUTER JOIN transportation_route_member_coalesced rm5 ON rm5.member = hl.osm_id AND rm5.concurrency_index=5
+ LEFT OUTER JOIN transportation_route_member_coalesced rm6 ON rm6.member = hl.osm_id AND rm6.concurrency_index=6
WHERE (hl.name <> '' OR hl.ref <> '' OR rm1.ref <> '' OR rm1.network <> '')
AND hl.highway <> ''
) AS t;
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_network_osm_id_idx ON osm_transportation_name_network (osm_id);
-CREATE INDEX IF NOT EXISTS osm_transportation_name_network_name_ref_idx ON osm_transportation_name_network (coalesce(tags->'name', ''), coalesce(ref, ''));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_network_geometry_idx ON osm_transportation_name_network USING gist (geometry);
--- Improve performance of the sql in transportation/update_route_member.sql
-CREATE INDEX IF NOT EXISTS osm_highway_linestring_highway_partial_idx
- ON osm_highway_linestring (highway)
- WHERE highway IN ('motorway', 'trunk');
+-- Create Primary-Key for osm_transportation_name_network table
+DO $$
+BEGIN
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_network' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_network ADD PRIMARY KEY (osm_id);
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+-- Geometry Index
+CREATE INDEX IF NOT EXISTS osm_transportation_name_network_geometry_idx
+ ON osm_transportation_name_network USING gist (geometry);
-- etldoc: osm_highway_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z11
CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z11(
- geometry geometry,
- id SERIAL PRIMARY KEY,
+ geometry geometry('LineString'),
+ id SERIAL,
osm_id bigint,
+ source_ids bigint[],
highway character varying,
network character varying,
construction character varying,
@@ -100,9 +144,52 @@ CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z11(
layer integer
);
-INSERT INTO osm_transportation_merge_linestring_gen_z11(geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer)
-SELECT (ST_Dump(ST_LineMerge(ST_Collect(geometry)))).geom AS geometry,
- NULL::bigint AS osm_id,
+-- Create osm_transportation_merge_linestring_gen_z10 as a copy of osm_transportation_merge_linestring_gen_z11 but
+-- drop the "source_ids" column. This can be done because z10 and z9 tables are only simplified and not merged,
+-- therefore relations to sources are direct via the id column.
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z10
+ (LIKE osm_transportation_merge_linestring_gen_z11);
+ALTER TABLE osm_transportation_merge_linestring_gen_z10 DROP COLUMN IF EXISTS source_ids;
+
+-- Create osm_transportation_merge_linestring_gen_z9 as a copy of osm_transportation_merge_linestring_gen_z10
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z9
+ (LIKE osm_transportation_merge_linestring_gen_z10);
+
+-- Create OneToMany-Relation-Table storing relations of a Merged-LineString in table
+-- osm_transportation_merge_linestring_gen_z11 to Source-LineStrings from table osm_highway_linestring_gen_z11
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z11_source_ids(
+ id int,
+ source_id bigint,
+ PRIMARY KEY (id, source_id)
+);
+
+-- Index for storing OSM-IDs of Source-LineStrings
+CREATE UNIQUE INDEX IF NOT EXISTS osm_highway_linestring_gen_z11_osm_id_idx ON osm_highway_linestring_gen_z11 ("osm_id");
+
+-- Analyze created indexes
+ANALYZE osm_highway_linestring_gen_z11;
+
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_merge_linestring_gen_z11;
+TRUNCATE osm_transportation_merge_linestring_gen_z11_source_ids;
+
+-- Merge LineStrings from osm_highway_linestring_gen_z11 by grouping them and creating intersecting clusters of
+-- each group via ST_ClusterDBSCAN
+INSERT INTO osm_transportation_merge_linestring_gen_z11 (geometry, source_ids, highway, network, construction,
+ is_bridge, is_tunnel, is_ford, expressway, z_order,
+ bicycle, foot, horse, mtb_scale, sac_scale, access, toll,
+ layer)
+SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries to
+ -- merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+ -- more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(osm_id) as source_ids,
+ -- Temporary Merged-LineString to Source-LineStrings-ID column to store relations before they have been
+ -- intersected
highway,
network,
construction,
@@ -121,26 +208,134 @@ SELECT (ST_Dump(ST_LineMerge(ST_Collect(geometry)))).geom AS geometry,
ELSE NULL::text END AS access,
toll,
layer
-FROM osm_highway_linestring_gen_z11
--- mapping.yaml pre-filter: motorway/trunk/primary/secondary/tertiary, with _link variants, construction, ST_IsValid()
-GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer
-;
+FROM (
+ SELECT osm_highway_linestring_normalized_brunnel_z11.*,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle,
+ foot, horse, mtb_scale, sac_scale, access, toll, layer
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle,
+ foot, horse, mtb_scale, sac_scale, access, toll, layer
+ ) as cluster_group
+ FROM (
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ SELECT geometry,
+ osm_id,
+ highway,
+ network,
+ construction,
+ visible_brunnel(geometry, is_bridge, 11) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 11) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 11) AS is_ford,
+ expressway,
+ z_order,
+ bicycle,
+ foot,
+ horse,
+ mtb_scale,
+ sac_scale,
+ access,
+ toll,
+ visible_layer(geometry, layer, 11) AS layer
+ FROM osm_highway_linestring_gen_z11
+ ) osm_highway_linestring_normalized_brunnel_z11
+) q
+GROUP BY cluster_group, cluster, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway,
+ bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer;
+
+-- Geometry Index
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z11_geometry_idx
ON osm_transportation_merge_linestring_gen_z11 USING gist (geometry);
-
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z10
- (LIKE osm_transportation_merge_linestring_gen_z11);
-
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z9
- (LIKE osm_transportation_merge_linestring_gen_z10);
-
-
-CREATE OR REPLACE FUNCTION insert_transportation_merge_linestring_gen_z10(update_id bigint) RETURNS void AS
-$$
+-- Create Primary-Keys for osm_transportation_merge_linestring_gen_z11/z10/z9 tables
+DO $$
BEGIN
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z11' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z11 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z10' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z10 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z9' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z9 ADD PRIMARY KEY (id);
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Indexes which can be utilized during full-update for queries originating from
+-- insert_transportation_merge_linestring_gen_z10() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z11_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z11 (id)
+ WHERE highway NOT IN ('tertiary', 'tertiary_link', 'busway') AND
+ construction NOT IN ('tertiary', 'tertiary_link', 'busway');
+
+-- Analyze populated table with new indexes
+ANALYZE osm_transportation_merge_linestring_gen_z11;
+
+-- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources. This required because
+-- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+-- more than two LineStrings form an intersection or no intersection could be found.
+-- Execute after indexes have been created on osm_highway_linestring_gen_z11 to improve performance
+INSERT INTO osm_transportation_merge_linestring_gen_z11_source_ids (id, source_id)
+SELECT m.id, m.source_id
+FROM (
+ SELECT id, unnest(source_ids) AS source_id, geometry
+ FROM osm_transportation_merge_linestring_gen_z11
+) m
+JOIN osm_highway_linestring_gen_z11 s ON (m.source_id = s.osm_id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (id, source_id) DO NOTHING;
+
+-- Drop temporary Merged-LineString to Source-LineStrings-ID column
+ALTER TABLE osm_transportation_merge_linestring_gen_z11 DROP COLUMN IF EXISTS source_ids;
+
+CREATE SCHEMA IF NOT EXISTS transportation;
+
+CREATE TABLE IF NOT EXISTS transportation.changes_z9_z10
+(
+ is_old boolean,
+ id int,
+ PRIMARY KEY (is_old, id)
+);
+
+CREATE OR REPLACE FUNCTION insert_transportation_merge_linestring_gen_z10(full_update bool) RETURNS void AS
+$$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh transportation z9 10';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation.changes_z9_z10;
+ ANALYZE osm_transportation_merge_linestring_gen_z11;
+
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z10
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z9_z10
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z9_z10.is_old IS TRUE AND
+ transportation.changes_z9_z10.id = osm_transportation_merge_linestring_gen_z10.id
+ );
-- etldoc: osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z10
INSERT INTO osm_transportation_merge_linestring_gen_z10
@@ -150,9 +345,10 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ visible_brunnel(geometry, is_bridge, 11) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 11) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 11) AS is_ford,
expressway,
z_order,
bicycle,
@@ -162,15 +358,35 @@ BEGIN
sac_scale,
access,
toll,
- layer
+ visible_layer(geometry, layer, 11) AS layer
FROM osm_transportation_merge_linestring_gen_z11
- WHERE (update_id IS NULL OR id = update_id)
- AND highway NOT IN ('tertiary', 'tertiary_link', 'busway', 'bus_guideway')
- AND construction NOT IN ('tertiary', 'tertiary_link', 'busway', 'bus_guideway')
- ;
+ WHERE (full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z9_z10
+ WHERE transportation.changes_z9_z10.is_old IS FALSE AND
+ transportation.changes_z9_z10.id = osm_transportation_merge_linestring_gen_z11.id
+ ))
+ AND (
+ highway NOT IN ('tertiary', 'tertiary_link', 'busway', 'bus_guideway')
+ AND construction NOT IN ('tertiary', 'tertiary_link', 'busway', 'bus_guideway')
+ )
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order,
+ bicycle = excluded.bicycle, foot = excluded.foot, horse = excluded.horse,
+ mtb_scale = excluded.mtb_scale, sac_scale = excluded.sac_scale,
+ access = excluded.access, toll = excluded.toll, layer = excluded.layer;
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z9
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z9_z10
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z9_z10.is_old IS TRUE AND
+ transportation.changes_z9_z10.id = osm_transportation_merge_linestring_gen_z9.id
+ );
+
+ -- Analyze source table
+ ANALYZE osm_transportation_merge_linestring_gen_z10;
-- etldoc: osm_transportation_merge_linestring_gen_z10 -> osm_transportation_merge_linestring_gen_z9
INSERT INTO osm_transportation_merge_linestring_gen_z9
@@ -180,9 +396,10 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ visible_brunnel(geometry, is_bridge, 10) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 10) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 10) AS is_ford,
expressway,
z_order,
bicycle,
@@ -192,31 +409,46 @@ BEGIN
sac_scale,
access,
toll,
- layer
+ visible_layer(geometry, layer, 10) AS layer
FROM osm_transportation_merge_linestring_gen_z10
- WHERE (update_id IS NULL OR id = update_id)
- ;
+ WHERE full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z9_z10
+ WHERE transportation.changes_z9_z10.is_old IS FALSE AND
+ transportation.changes_z9_z10.id = osm_transportation_merge_linestring_gen_z10.id
+ )
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order,
+ bicycle = excluded.bicycle, foot = excluded.foot, horse = excluded.horse,
+ mtb_scale = excluded.mtb_scale, sac_scale = excluded.sac_scale,
+ access = excluded.access, toll = excluded.toll, layer = excluded.layer;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation.changes_z9_z10;
+
+ RAISE LOG 'Refresh transportation z9 10 done in %', age(clock_timestamp(), t);
END;
$$ LANGUAGE plpgsql;
-SELECT insert_transportation_merge_linestring_gen_z10(NULL);
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_merge_linestring_gen_z10;
+TRUNCATE osm_transportation_merge_linestring_gen_z9;
+SELECT insert_transportation_merge_linestring_gen_z10(TRUE);
+
+-- Geometry Indexes
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z10_geometry_idx
ON osm_transportation_merge_linestring_gen_z10 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z10_id_idx
- ON osm_transportation_merge_linestring_gen_z10(id);
-
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z9_geometry_idx
ON osm_transportation_merge_linestring_gen_z9 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z9_id_idx
- ON osm_transportation_merge_linestring_gen_z9(id);
-
-- etldoc: osm_transportation_merge_linestring_gen_z9 -> osm_transportation_merge_linestring_gen_z8
CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z8(
- geometry geometry,
- id SERIAL PRIMARY KEY,
+ geometry geometry('LineString'),
+ id SERIAL,
osm_id bigint,
+ source_ids int[],
highway character varying,
network character varying,
construction character varying,
@@ -227,9 +459,64 @@ CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z8(
z_order integer
);
-INSERT INTO osm_transportation_merge_linestring_gen_z8(geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order)
-SELECT ST_Simplify(ST_LineMerge(ST_Collect(geometry)), ZRes(10)) AS geometry,
- NULL::bigint AS osm_id,
+-- Create osm_transportation_merge_linestring_gen_z7 as a copy of osm_transportation_merge_linestring_gen_z8 but
+-- drop the "source_ids" column. This can be done because z7 to z5 tables are only simplified and not merged,
+-- therefore relations to sources are direct via the id column.
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z7
+ (LIKE osm_transportation_merge_linestring_gen_z8);
+ALTER TABLE osm_transportation_merge_linestring_gen_z7 DROP COLUMN IF EXISTS source_ids;
+
+-- Create osm_transportation_merge_linestring_gen_z6 as a copy of osm_transportation_merge_linestring_gen_z7
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z6
+ (LIKE osm_transportation_merge_linestring_gen_z7);
+
+-- Create osm_transportation_merge_linestring_gen_z5 as a copy of osm_transportation_merge_linestring_gen_z6
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z5
+ (LIKE osm_transportation_merge_linestring_gen_z6);
+
+-- Create osm_transportation_merge_linestring_gen_z4 as a copy of osm_transportation_merge_linestring_gen_z5
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z4
+ (LIKE osm_transportation_merge_linestring_gen_z5);
+
+-- Create OneToMany-Relation-Table storing relations of a Merged-LineString in table
+-- osm_transportation_merge_linestring_gen_z8 to Source-LineStrings from table
+-- osm_transportation_merge_linestring_gen_z9
+CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z8_source_ids(
+ id int,
+ source_id bigint,
+ PRIMARY KEY (id, source_id)
+);
+
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_merge_linestring_gen_z8;
+TRUNCATE osm_transportation_merge_linestring_gen_z8_source_ids;
+
+-- Indexes for filling and updating osm_transportation_merge_linestring_gen_z8 table
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z9_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z9 (id)
+ WHERE (
+ highway IN ('motorway', 'trunk', 'primary') OR
+ construction IN ('motorway', 'trunk', 'primary')
+ ) AND ST_IsValid(geometry) AND access IS NULL;
+
+-- Analyze populated table with indexes
+ANALYZE osm_transportation_merge_linestring_gen_z9;
+
+-- Merge LineStrings from osm_transportation_merge_linestring_gen_z9 by grouping them and creating intersecting
+-- clusters of each group via ST_ClusterDBSCAN
+INSERT INTO osm_transportation_merge_linestring_gen_z8(geometry, source_ids, highway, network, construction, is_bridge,
+ is_tunnel, is_ford, expressway, z_order)
+SELECT (ST_Dump(ST_Simplify(ST_LineMerge(ST_Union(geometry)), ZRes(10)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries to
+ -- merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+ -- more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(id) AS source_ids,
+ -- Temporary Merged-LineString to Source-LineStrings-ID column to store relations before they have been
+ -- intersected
highway,
network,
construction,
@@ -238,34 +525,140 @@ SELECT ST_Simplify(ST_LineMerge(ST_Collect(geometry)), ZRes(10)) AS geometry,
is_ford,
expressway,
min(z_order) as z_order
-FROM osm_transportation_merge_linestring_gen_z9
-WHERE (highway IN ('motorway', 'trunk', 'primary') OR
- construction IN ('motorway', 'trunk', 'primary'))
- AND ST_IsValid(geometry)
- AND access IS NULL
-GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
-;
+FROM (
+ SELECT osm_highway_linestring_normalized_brunnel_z9.*,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
+ ) as cluster_group
+ FROM (
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ SELECT id,
+ geometry,
+ highway,
+ network,
+ construction,
+ visible_brunnel(geometry, is_bridge, 9) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 9) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 9) AS is_ford,
+ expressway,
+ z_order
+ FROM osm_transportation_merge_linestring_gen_z9
+ WHERE (
+ highway IN ('motorway', 'trunk', 'primary') OR
+ construction IN ('motorway', 'trunk', 'primary')
+ ) AND ST_IsValid(geometry) AND access IS NULL
+ ) osm_highway_linestring_normalized_brunnel_z9
+) q
+GROUP BY cluster_group, cluster, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway;
+
+-- Geometry Index
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z8_geometry_idx
ON osm_transportation_merge_linestring_gen_z8 USING gist (geometry);
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z7
- (LIKE osm_transportation_merge_linestring_gen_z8);
-
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z6
- (LIKE osm_transportation_merge_linestring_gen_z7);
-
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z5
- (LIKE osm_transportation_merge_linestring_gen_z6);
-
-CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z4
- (LIKE osm_transportation_merge_linestring_gen_z5);
-
-
-CREATE OR REPLACE FUNCTION insert_transportation_merge_linestring_gen_z7(update_id bigint) RETURNS void AS
-$$
+-- Create Primary-Keys for osm_transportation_merge_linestring_gen_z8/z7/z6/z5/z4 tables
+DO $$
BEGIN
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z8' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z8 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z7' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z7 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z6' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z6 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z5' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z5 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_merge_linestring_gen_z4' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_merge_linestring_gen_z4 ADD PRIMARY KEY (id);
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Indexes which can be utilized during full-update for queries originating from
+-- insert_transportation_merge_linestring_gen_z7() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z8_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z8 (id)
+ WHERE ST_Length(geometry) > 50;
+
+-- Analyze populated table with indexes
+ANALYZE osm_transportation_merge_linestring_gen_z8;
+
+-- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources. This required because
+-- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+-- more than two LineStrings form an intersection or no intersection could be found.
+-- Execute after indexes have been created on osm_transportation_merge_linestring_gen_z11 to improve performance
+INSERT INTO osm_transportation_merge_linestring_gen_z8_source_ids (id, source_id)
+SELECT m.id, m.source_id
+FROM (
+ SELECT id, unnest(source_ids) AS source_id, geometry
+ FROM osm_transportation_merge_linestring_gen_z8
+) m
+JOIN osm_transportation_merge_linestring_gen_z9 s ON (m.source_id = s.id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (id, source_id) DO NOTHING;
+
+-- Drop temporary Merged-LineString to Source-LineStrings-ID column
+ALTER TABLE osm_transportation_merge_linestring_gen_z8 DROP COLUMN IF EXISTS source_ids;
+
+CREATE TABLE IF NOT EXISTS transportation.changes_z4_z5_z6_z7
+(
+ is_old boolean,
+ id int,
+ PRIMARY KEY (is_old, id)
+);
+
+CREATE OR REPLACE FUNCTION insert_transportation_merge_linestring_gen_z7(full_update boolean) RETURNS void AS
+$$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh transportation z4 z5 z6 z7';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation.changes_z4_z5_z6_z7;
+ ANALYZE osm_transportation_merge_linestring_gen_z8;
+
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z7
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z4_z5_z6_z7
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z4_z5_z6_z7.is_old IS TRUE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z7.id
+ );
-- etldoc: osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z7
INSERT INTO osm_transportation_merge_linestring_gen_z7
@@ -275,19 +668,36 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ visible_brunnel(geometry, is_bridge, 8) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 8) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 8) AS is_ford,
expressway,
z_order
FROM osm_transportation_merge_linestring_gen_z8
-- Current view: motorway/trunk/primary
WHERE
- (update_id IS NULL OR id = update_id) AND
- ST_Length(geometry) > 50;
+ (full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z4_z5_z6_z7
+ WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z8.id
+ )) AND
+ (ST_Length(geometry) > 50)
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order;
+ -- Analyze source table
+ ANALYZE osm_transportation_merge_linestring_gen_z7;
+
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z6
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z4_z5_z6_z7
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z4_z5_z6_z7.is_old IS TRUE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z6.id
+ );
-- etldoc: osm_transportation_merge_linestring_gen_z7 -> osm_transportation_merge_linestring_gen_z6
INSERT INTO osm_transportation_merge_linestring_gen_z6
@@ -297,19 +707,37 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ visible_brunnel(geometry, is_bridge, 7) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 7) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 7) AS is_ford,
expressway,
z_order
FROM osm_transportation_merge_linestring_gen_z7
+ -- Current view: motorway/trunk/primary
WHERE
- (update_id IS NULL OR id = update_id) AND
+ (full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z4_z5_z6_z7
+ WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z7.id
+ )) AND
(highway IN ('motorway', 'trunk') OR construction IN ('motorway', 'trunk')) AND
- ST_Length(geometry) > 100;
+ ST_Length(geometry) > 100
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order;
+ -- Analyze source table
+ ANALYZE osm_transportation_merge_linestring_gen_z6;
+
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z5
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z4_z5_z6_z7
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z4_z5_z6_z7.is_old IS TRUE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z5.id
+ );
-- etldoc: osm_transportation_merge_linestring_gen_z6 -> osm_transportation_merge_linestring_gen_z5
INSERT INTO osm_transportation_merge_linestring_gen_z5
@@ -319,24 +747,41 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+ visible_brunnel(geometry, is_bridge, 6) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 6) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 6) AS is_ford,
expressway,
z_order
FROM osm_transportation_merge_linestring_gen_z6
WHERE
- (update_id IS NULL OR id = update_id) AND
+ (full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z4_z5_z6_z7
+ WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z6.id
+ )) AND
-- Current view: all motorways and trunks of national-importance
(highway = 'motorway'
OR construction = 'motorway'
- -- Allow trunk roads that are part of a nation's most important route network to show at z4
+ -- Allow trunk roads that are part of a nation's most important route network to show at z5
OR (highway = 'trunk' AND osm_national_network(network))
) AND
- ST_Length(geometry) > 500;
+ ST_Length(geometry) > 500
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order;
+ -- Analyze source table
+ ANALYZE osm_transportation_merge_linestring_gen_z5;
+
+ -- Remove entries which have been deleted from source table
DELETE FROM osm_transportation_merge_linestring_gen_z4
- WHERE update_id IS NULL OR id = update_id;
+ USING transportation.changes_z4_z5_z6_z7
+ WHERE full_update IS TRUE OR (
+ transportation.changes_z4_z5_z6_z7.is_old IS TRUE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z4.id
+ );
-- etldoc: osm_transportation_merge_linestring_gen_z5 -> osm_transportation_merge_linestring_gen_z4
INSERT INTO osm_transportation_merge_linestring_gen_z4
@@ -346,90 +791,116 @@ BEGIN
highway,
network,
construction,
- is_bridge,
- is_tunnel,
- is_ford,
+ visible_brunnel(geometry, is_bridge, 5) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 5) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 5) AS is_ford,
expressway,
z_order
FROM osm_transportation_merge_linestring_gen_z5
WHERE
- (update_id IS NULL OR id = update_id) AND
+ (full_update IS TRUE OR EXISTS(
+ SELECT NULL FROM transportation.changes_z4_z5_z6_z7
+ WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
+ transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z5.id
+ )) AND
osm_national_network(network) AND
-- Current view: national-importance motorways and trunks
- ST_Length(geometry) > 1000;
+ ST_Length(geometry) > 1000
+ ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
+ construction = excluded.construction, is_bridge = excluded.is_bridge,
+ is_tunnel = excluded.is_tunnel, is_ford = excluded.is_ford,
+ expressway = excluded.expressway, z_order = excluded.z_order;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation.changes_z4_z5_z6_z7;
+
+ RAISE LOG 'Refresh transportation z4 z5 z6 z7 done in %', age(clock_timestamp(), t);
END;
$$ LANGUAGE plpgsql;
-SELECT insert_transportation_merge_linestring_gen_z7(NULL);
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_merge_linestring_gen_z7;
+TRUNCATE osm_transportation_merge_linestring_gen_z6;
+TRUNCATE osm_transportation_merge_linestring_gen_z5;
+TRUNCATE osm_transportation_merge_linestring_gen_z4;
+SELECT insert_transportation_merge_linestring_gen_z7(TRUE);
+
+-- Indexes for queries originating from insert_transportation_merge_linestring_gen_z7() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z7_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z7 (id)
+ WHERE (highway IN ('motorway', 'trunk') OR construction IN ('motorway', 'trunk')) AND
+ ST_Length(geometry) > 100;
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z6_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z6 (id)
+ WHERE (highway = 'motorway'
+ OR construction = 'motorway'
+ OR (highway = 'trunk' AND osm_national_network(network))
+ ) AND
+ ST_Length(geometry) > 500;
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z5_update_partial_idx
+ ON osm_transportation_merge_linestring_gen_z5 (id)
+ WHERE osm_national_network(network) AND ST_Length(geometry) > 1000;
+
+-- Geometry Indexes
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z7_geometry_idx
ON osm_transportation_merge_linestring_gen_z7 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z7_id_idx
- ON osm_transportation_merge_linestring_gen_z7(id);
-
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z6_geometry_idx
ON osm_transportation_merge_linestring_gen_z6 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z6_id_idx
- ON osm_transportation_merge_linestring_gen_z6(id);
-
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z5_geometry_idx
ON osm_transportation_merge_linestring_gen_z5 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z5_id_idx
- ON osm_transportation_merge_linestring_gen_z5(id);
-
CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z4_geometry_idx
ON osm_transportation_merge_linestring_gen_z4 USING gist (geometry);
-CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z4_id_idx
- ON osm_transportation_merge_linestring_gen_z4(id);
-- Handle updates on
-- osm_highway_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z11
-
-CREATE SCHEMA IF NOT EXISTS transportation;
+-- osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z10
+-- osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z9
+CREATE OR REPLACE AGGREGATE array_cat_agg(anycompatiblearray) (
+ SFUNC=array_cat,
+ STYPE=anycompatiblearray,
+ INITCOND = '{}'
+);
CREATE TABLE IF NOT EXISTS transportation.changes_z11
(
- id serial PRIMARY KEY,
- is_old boolean,
- geometry geometry,
+ is_old boolean NULL,
osm_id bigint,
- highway character varying,
- network character varying,
- construction character varying,
- is_bridge boolean,
- is_tunnel boolean,
- is_ford boolean,
- expressway boolean,
- z_order integer,
- bicycle character varying,
- foot character varying,
- horse character varying,
- mtb_scale character varying,
- sac_scale character varying,
- access character varying,
- toll boolean,
- layer integer
+ PRIMARY KEY (is_old, osm_id)
);
-CREATE OR REPLACE FUNCTION transportation.store_z11() RETURNS trigger AS
+-- Store IDs of changed elements from osm_highway_linestring_gen_z11 table.
+CREATE OR REPLACE FUNCTION transportation.store_gen_z11() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE' OR tg_op = 'UPDATE') THEN
- INSERT INTO transportation.changes_z11(is_old, geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer)
- VALUES (true, old.geometry, old.osm_id, old.highway, old.network, old.construction, old.is_bridge, old.is_tunnel, old.is_ford, old.expressway, old.z_order, old.bicycle, old.foot, old.horse, old.mtb_scale, old.sac_scale,
- CASE
- WHEN old.access IN ('private', 'no') THEN 'no'
- ELSE NULL::text END,
- old.toll, old.layer);
+ IF (tg_op = 'INSERT' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z11(is_old, osm_id)
+ VALUES (FALSE, new.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
END IF;
- IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
- INSERT INTO transportation.changes_z11(is_old, geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer)
- VALUES (false, new.geometry, new.osm_id, new.highway, new.network, new.construction, new.is_bridge, new.is_tunnel, new.is_ford, new.expressway, new.z_order, new.bicycle, new.foot, new.horse, new.mtb_scale, new.sac_scale,
- CASE
- WHEN new.access IN ('private', 'no') THEN 'no'
- ELSE NULL::text END,
- new.toll, new.layer);
+ IF (tg_op = 'DELETE' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z11(is_old, osm_id)
+ VALUES (TRUE, old.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Store IDs of changed elements from osm_highway_linestring_gen_z9 table.
+CREATE OR REPLACE FUNCTION transportation.store_merge_z11() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op = 'INSERT' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z9_z10(is_old, id)
+ VALUES (FALSE, new.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
+ END IF;
+ IF tg_op = 'DELETE' THEN
+ INSERT INTO transportation.changes_z9_z10(is_old, id)
+ VALUES (TRUE, old.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
END IF;
RETURN NULL;
END;
@@ -456,166 +927,218 @@ DECLARE
BEGIN
RAISE LOG 'Refresh transportation z11';
- -- Compact the change history to keep only the first and last version
- CREATE TEMP TABLE changes_compact AS
- SELECT
- *
- FROM ((
- SELECT DISTINCT ON (osm_id) *
- FROM transportation.changes_z11
- WHERE is_old
- ORDER BY osm_id,
- id ASC
- ) UNION ALL (
- SELECT DISTINCT ON (osm_id) *
- FROM transportation.changes_z11
- WHERE NOT is_old
- ORDER BY osm_id,
- id DESC
- )) AS t;
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation.changes_z11;
+ ANALYZE osm_highway_linestring_gen_z11;
- -- Collect all original existing ways from impacted mmerge
- CREATE TEMP TABLE osm_highway_linestring_original AS
- SELECT DISTINCT ON (h.osm_id)
- NULL::integer AS id,
- NULL::boolean AS is_old,
- h.geometry,
- h.osm_id,
- h.highway,
- h.network,
- h.construction,
- h.is_bridge,
- h.is_tunnel,
- h.is_ford,
- h.expressway,
- h.z_order,
- h.bicycle,
- h.foot,
- h.horse,
- h.mtb_scale,
- h.sac_scale,
- h.access,
- h.toll,
- h.layer
- FROM
- changes_compact AS c
- JOIN osm_transportation_merge_linestring_gen_z11 AS m ON
- m.geometry && c.geometry
- AND m.highway IS NOT DISTINCT FROM c.highway
- AND m.network IS NOT DISTINCT FROM c.network
- AND m.construction IS NOT DISTINCT FROM c.construction
- AND m.is_bridge IS NOT DISTINCT FROM c.is_bridge
- AND m.is_tunnel IS NOT DISTINCT FROM c.is_tunnel
- AND m.is_ford IS NOT DISTINCT FROM c.is_ford
- AND m.expressway IS NOT DISTINCT FROM c.expressway
- AND m.bicycle IS NOT DISTINCT FROM c.bicycle
- AND m.foot IS NOT DISTINCT FROM c.foot
- AND m.horse IS NOT DISTINCT FROM c.horse
- AND m.mtb_scale IS NOT DISTINCT FROM c.mtb_scale
- AND m.sac_scale IS NOT DISTINCT FROM c.sac_scale
- AND m.access IS NOT DISTINCT FROM c.access
- AND m.toll IS NOT DISTINCT FROM c.toll
- AND m.layer IS NOT DISTINCT FROM c.layer
- JOIN osm_highway_linestring_gen_z11 AS h ON
- h.geometry && c.geometry
- AND h.osm_id NOT IN (SELECT osm_id FROM changes_compact)
- AND ST_Contains(m.geometry, h.geometry)
- AND h.highway IS NOT DISTINCT FROM m.highway
- AND h.network IS NOT DISTINCT FROM m.network
- AND h.construction IS NOT DISTINCT FROM m.construction
- AND h.is_bridge IS NOT DISTINCT FROM m.is_bridge
- AND h.is_tunnel IS NOT DISTINCT FROM m.is_tunnel
- AND h.is_ford IS NOT DISTINCT FROM m.is_ford
- AND h.expressway IS NOT DISTINCT FROM m.expressway
- AND h.bicycle IS NOT DISTINCT FROM m.bicycle
- AND h.foot IS NOT DISTINCT FROM m.foot
- AND h.horse IS NOT DISTINCT FROM m.horse
- AND h.mtb_scale IS NOT DISTINCT FROM m.mtb_scale
- AND h.sac_scale IS NOT DISTINCT FROM m.sac_scale
- AND CASE
- WHEN h.access IN ('private', 'no') THEN 'no'
- ELSE NULL::text END IS NOT DISTINCT FROM m.access
- AND h.toll IS NOT DISTINCT FROM m.toll
- AND h.layer IS NOT DISTINCT FROM m.layer
- ORDER BY
- h.osm_id
- ;
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_transportation_merge_linestring_gen_z11_source_ids m
+ WHERE EXISTS(
+ SELECT NULL
+ FROM transportation.changes_z11 c
+ WHERE c.is_old IS TRUE AND c.osm_id = m.source_id
+ )
+ GROUP BY id;
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
DELETE
- FROM osm_transportation_merge_linestring_gen_z11 AS m
- USING changes_compact AS c
- WHERE
- m.geometry && c.geometry
- AND m.highway IS NOT DISTINCT FROM c.highway
- AND m.network IS NOT DISTINCT FROM c.network
- AND m.construction IS NOT DISTINCT FROM c.construction
- AND m.is_bridge IS NOT DISTINCT FROM c.is_bridge
- AND m.is_tunnel IS NOT DISTINCT FROM c.is_tunnel
- AND m.is_ford IS NOT DISTINCT FROM c.is_ford
- AND m.expressway IS NOT DISTINCT FROM c.expressway
- AND m.bicycle IS NOT DISTINCT FROM c.bicycle
- AND m.foot IS NOT DISTINCT FROM c.foot
- AND m.horse IS NOT DISTINCT FROM c.horse
- AND m.mtb_scale IS NOT DISTINCT FROM c.mtb_scale
- AND m.sac_scale IS NOT DISTINCT FROM c.sac_scale
- AND m.access IS NOT DISTINCT FROM c.access
- AND m.toll IS NOT DISTINCT FROM c.toll
- AND m.layer IS NOT DISTINCT FROM c.layer
- ;
+ FROM osm_transportation_merge_linestring_gen_z11 m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z11_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
- INSERT INTO osm_transportation_merge_linestring_gen_z11(geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer)
- SELECT (ST_Dump(ST_LineMerge(ST_Collect(geometry)))).geom AS geometry,
- NULL::bigint AS osm_id,
- highway,
- network,
- construction,
- is_bridge,
- is_tunnel,
- is_ford,
- expressway,
- min(z_order) as z_order,
- bicycle,
- foot,
- horse,
- mtb_scale,
- sac_scale,
- CASE
- WHEN access IN ('private', 'no') THEN 'no'
- ELSE NULL::text END AS access,
- toll,
- layer
- FROM ((
- SELECT * FROM osm_highway_linestring_original
- ) UNION ALL (
- -- New or updated ways
- SELECT
- *
- FROM
- changes_compact
- WHERE
- NOT is_old
- )) AS t
- GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer
- ;
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_transportation_merge_linestring_gen_z11;
+ ANALYZE osm_transportation_merge_linestring_gen_z11_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT osm_highway_linestring_gen_z11.osm_id, NULL::INTEGER AS id, NULL::BIGINT[] AS source_ids,
+ geometry, highway, network, construction,
+ visible_brunnel(geometry, is_bridge, 11) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 11) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 11) AS is_ford,
+ expressway, bicycle, foot, horse, mtb_scale, sac_scale,
+ CASE WHEN access IN ('private', 'no') THEN 'no' ELSE NULL::text END AS access, toll,
+ visible_layer(geometry, layer, 11) AS layer, z_order
+ -- Table containing the IDs of all Source-LineStrings affected by this update
+ FROM (
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT osm_id AS source_id FROM transportation.changes_z11 WHERE is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_highway_linestring_gen_z11 ON (
+ affected_source_linestrings.source_id = osm_highway_linestring_gen_z11.osm_id
+ );
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::BIGINT AS osm_id, m.id,
+ ARRAY(
+ SELECT s.source_id FROM osm_transportation_merge_linestring_gen_z11_source_ids s WHERE s.id = m.id
+ )::BIGINT[] AS source_ids, m.geometry, m.highway, m.network, m.construction,
+ visible_brunnel(m.geometry, m.is_bridge, 11) AS is_bridge,
+ visible_brunnel(m.geometry, m.is_tunnel, 11) AS is_tunnel,
+ visible_brunnel(m.geometry, m.is_ford, 11) AS is_ford,
+ m.expressway, m.bicycle, m.foot, m.horse, m.mtb_scale, m.sac_scale, m.access, m.toll,
+ visible_layer(m.geometry, m.layer, 11) AS layer, m.z_order
+ FROM linestrings_to_merge
+ JOIN osm_transportation_merge_linestring_gen_z11 m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry));
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z11 m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z11_source_ids m
+ USING linestrings_to_merge
+ WHERE linestrings_to_merge.id = m.id;
+
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle, foot,
+ horse, mtb_scale, sac_scale, access, toll, layer
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, bicycle, foot, horse,
+ mtb_scale, sac_scale, access, toll, layer
+ ) as cluster_group
+ FROM linestrings_to_merge;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
+
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
+
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_transportation_merge_linestring_gen_z11 ADD COLUMN IF NOT EXISTS new_source_ids BIGINT[];
+ ALTER TABLE osm_transportation_merge_linestring_gen_z11 ADD COLUMN IF NOT EXISTS old_source_ids BIGINT[];
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_transportation_merge_linestring_gen_z11(geometry, new_source_ids, old_source_ids, highway,
+ network, construction, is_bridge, is_tunnel, is_ford,
+ expressway, z_order, bicycle, foot, horse, mtb_scale,
+ sac_scale, access, toll, layer)
+ SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries to
+ -- merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+ -- more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(osm_id) FILTER (WHERE osm_id IS NOT NULL), '{}' )::BIGINT[] AS new_source_ids,
+ array_cat_agg(source_ids)::BIGINT[] AS old_source_ids,
+ highway,
+ network,
+ construction,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ expressway,
+ min(z_order) as z_order,
+ bicycle,
+ foot,
+ horse,
+ mtb_scale,
+ sac_scale,
+ access,
+ toll,
+ layer
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway,
+ bicycle, foot, horse, mtb_scale, sac_scale, access, toll, layer
+ RETURNING id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources.
+ -- This is required because ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_transportation_merge_linestring_gen_z11_source_ids (id, source_id)
+ SELECT m.id, source_id
+ FROM (
+ SELECT id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_highway_linestring_gen_z11 s ON (m.source_id = s.osm_id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (id, source_id) DO NOTHING;
+
+ -- Cleanup remaining table
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_transportation_merge_linestring_gen_z11 DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_transportation_merge_linestring_gen_z11 DROP COLUMN IF EXISTS old_source_ids;
- DROP TABLE osm_highway_linestring_original;
- DROP TABLE changes_compact;
-- noinspection SqlWithoutWhere
DELETE FROM transportation.changes_z11;
-- noinspection SqlWithoutWhere
DELETE FROM transportation.updates_z11;
RAISE LOG 'Refresh transportation z11 done in %', age(clock_timestamp(), t);
+
+ -- Update z10 and z9 tables
+ PERFORM insert_transportation_merge_linestring_gen_z10(FALSE);
+
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
-
CREATE TRIGGER trigger_store_transportation_highway_linestring_gen_z11
AFTER INSERT OR UPDATE OR DELETE
ON osm_highway_linestring_gen_z11
FOR EACH ROW
-EXECUTE PROCEDURE transportation.store_z11();
+EXECUTE PROCEDURE transportation.store_gen_z11();
+
+CREATE TRIGGER trigger_store_osm_transportation_merge_linestring_gen_z11
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_transportation_merge_linestring_gen_z11
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation.store_merge_z11();
CREATE TRIGGER trigger_flag_transportation_z11
AFTER INSERT OR UPDATE OR DELETE
@@ -631,64 +1154,51 @@ CREATE CONSTRAINT TRIGGER trigger_refresh_z11
EXECUTE PROCEDURE transportation.refresh_z11();
--- Handle updates on
--- osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z10
--- osm_transportation_merge_linestring_gen_z11 -> osm_transportation_merge_linestring_gen_z9
-
-
-CREATE OR REPLACE FUNCTION transportation.merge_linestring_gen_refresh_z10() RETURNS trigger AS
-$$
-BEGIN
- IF (tg_op = 'DELETE') THEN
- DELETE FROM osm_transportation_merge_linestring_gen_z10 WHERE id = old.id;
- DELETE FROM osm_transportation_merge_linestring_gen_z9 WHERE id = old.id;
- END IF;
-
- IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
- PERFORM insert_transportation_merge_linestring_gen_z10(new.id);
- END IF;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-
-CREATE TRIGGER trigger_osm_transportation_merge_linestring_gen_z11
- AFTER INSERT OR UPDATE OR DELETE
- ON osm_transportation_merge_linestring_gen_z11
- FOR EACH ROW
-EXECUTE PROCEDURE transportation.merge_linestring_gen_refresh_z10();
-
-
-- Handle updates on
-- osm_transportation_merge_linestring_gen_z9 -> osm_transportation_merge_linestring_gen_z8
-
+-- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z7
+-- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z6
+-- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z5
+-- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z4
CREATE TABLE IF NOT EXISTS transportation.changes_z9
(
is_old boolean,
- geometry geometry,
id bigint,
- highway character varying,
- network character varying,
- construction character varying,
- is_bridge boolean,
- is_tunnel boolean,
- is_ford boolean,
- expressway boolean,
- z_order integer
+ PRIMARY KEY (is_old, id)
);
+-- Store IDs of changed elements from osm_highway_linestring_gen_z9 table.
CREATE OR REPLACE FUNCTION transportation.store_z9() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE' OR tg_op = 'UPDATE') THEN
- INSERT INTO transportation.changes_z9(is_old, geometry, id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order)
- VALUES (true, old.geometry, old.id, old.highway, old.network, old.construction, old.is_bridge, old.is_tunnel, old.is_ford, old.expressway, old.z_order);
+ IF (tg_op = 'INSERT' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z9(is_old, id)
+ VALUES (FALSE, new.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
END IF;
- IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
- INSERT INTO transportation.changes_z9(is_old, geometry, id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order)
- VALUES (false, new.geometry, new.id, new.highway, new.network, new.construction, new.is_bridge, new.is_tunnel, new.is_ford, new.expressway, new.z_order);
+ IF (tg_op = 'DELETE' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z9(is_old, id)
+ VALUES (TRUE, old.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Store IDs of changed elements from osm_highway_linestring_gen_z8 table.
+CREATE OR REPLACE FUNCTION transportation.store_z8() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op = 'INSERT' OR tg_op = 'UPDATE') THEN
+ INSERT INTO transportation.changes_z4_z5_z6_z7(is_old, id)
+ VALUES (FALSE, new.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
+ END IF;
+ IF tg_op = 'DELETE' THEN
+ INSERT INTO transportation.changes_z4_z5_z6_z7(is_old, id)
+ VALUES (TRUE, old.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
END IF;
RETURN NULL;
END;
@@ -713,124 +1223,209 @@ $$
DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
- RAISE LOG 'Refresh transportation z9';
+ RAISE LOG 'Refresh transportation z8';
- -- Compact the change history to keep only the first and last version
- CREATE TEMP TABLE changes_compact AS
- SELECT
- *
- FROM ((
- SELECT DISTINCT ON (id) *
- FROM transportation.changes_z9
- WHERE is_old
- ORDER BY id,
- id ASC
- ) UNION ALL (
- SELECT DISTINCT ON (id) *
- FROM transportation.changes_z9
- WHERE NOT is_old
- ORDER BY id,
- id DESC
- )) AS t;
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation.changes_z9;
+ ANALYZE osm_transportation_merge_linestring_gen_z9;
- -- Collect all original existing ways from impacted mmerge
- CREATE TEMP TABLE osm_highway_linestring_original AS
- SELECT DISTINCT ON (h.id)
- NULL::boolean AS is_old,
- h.geometry,
- h.id,
- h.highway,
- h.network,
- h.construction,
- h.is_bridge,
- h.is_tunnel,
- h.is_ford,
- h.expressway,
- h.z_order
- FROM
- changes_compact AS c
- JOIN osm_transportation_merge_linestring_gen_z8 AS m ON
- m.geometry && c.geometry
- AND m.highway IS NOT DISTINCT FROM c.highway
- AND m.network IS NOT DISTINCT FROM c.network
- AND m.construction IS NOT DISTINCT FROM c.construction
- AND m.is_bridge IS NOT DISTINCT FROM c.is_bridge
- AND m.is_tunnel IS NOT DISTINCT FROM c.is_tunnel
- AND m.is_ford IS NOT DISTINCT FROM c.is_ford
- AND m.expressway IS NOT DISTINCT FROM c.expressway
- JOIN osm_transportation_merge_linestring_gen_z9 AS h ON
- h.geometry && c.geometry
- AND h.id NOT IN (SELECT id FROM changes_compact)
- AND ST_Contains(m.geometry, h.geometry)
- AND h.highway IS NOT DISTINCT FROM m.highway
- AND h.network IS NOT DISTINCT FROM m.network
- AND h.construction IS NOT DISTINCT FROM m.construction
- AND h.is_bridge IS NOT DISTINCT FROM m.is_bridge
- AND h.is_tunnel IS NOT DISTINCT FROM m.is_tunnel
- AND h.is_ford IS NOT DISTINCT FROM m.is_ford
- AND h.expressway IS NOT DISTINCT FROM m.expressway
- ORDER BY
- h.id
- ;
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_transportation_merge_linestring_gen_z8_source_ids m
+ WHERE EXISTS(
+ SELECT NULL
+ FROM transportation.changes_z9 c
+ WHERE c.is_old IS TRUE AND c.id = m.source_id
+ )
+ GROUP BY id;
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
DELETE
- FROM osm_transportation_merge_linestring_gen_z8 AS m
- USING changes_compact AS c
- WHERE
- m.geometry && c.geometry
- AND m.highway IS NOT DISTINCT FROM c.highway
- AND m.network IS NOT DISTINCT FROM c.network
- AND m.construction IS NOT DISTINCT FROM c.construction
- AND m.is_bridge IS NOT DISTINCT FROM c.is_bridge
- AND m.is_tunnel IS NOT DISTINCT FROM c.is_tunnel
- AND m.is_ford IS NOT DISTINCT FROM c.is_ford
- AND m.expressway IS NOT DISTINCT FROM c.expressway
- ;
+ FROM osm_transportation_merge_linestring_gen_z8 m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z8_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
- INSERT INTO osm_transportation_merge_linestring_gen_z8(geometry, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order)
- SELECT (ST_Dump(ST_LineMerge(ST_Collect(geometry)))).geom AS geometry,
- NULL::bigint AS osm_id,
- highway,
- network,
- construction,
- is_bridge,
- is_tunnel,
- is_ford,
- expressway,
- min(z_order) as z_order
- FROM ((
- SELECT * FROM osm_highway_linestring_original
- ) UNION ALL (
- -- New or updated ways
- SELECT
- *
- FROM
- changes_compact
- WHERE
- NOT is_old
- )) AS t
- GROUP BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
- ;
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_transportation_merge_linestring_gen_z8;
+ ANALYZE osm_transportation_merge_linestring_gen_z8_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT id AS source_id, NULL::INT AS id, NULL::INT[] AS source_ids, geometry, highway, network, construction,
+ visible_brunnel(geometry, is_bridge, 9) AS is_bridge,
+ visible_brunnel(geometry, is_tunnel, 9) AS is_tunnel,
+ visible_brunnel(geometry, is_ford, 9) AS is_ford, expressway, z_order
+ -- Create a table containing the IDs of all Source-LineStrings affected by this update
+ FROM (
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT id AS source_id FROM transportation.changes_z9 WHERE transportation.changes_z9.is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_transportation_merge_linestring_gen_z9 ON (
+ affected_source_linestrings.source_id = osm_transportation_merge_linestring_gen_z9.id
+ )
+ WHERE (
+ highway IN ('motorway', 'trunk', 'primary') OR
+ construction IN ('motorway', 'trunk', 'primary')
+ ) AND
+ ST_IsValid(geometry) AND
+ access IS NULL;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::INT AS source_id, m.id,
+ ARRAY(
+ SELECT s.source_id FROM osm_transportation_merge_linestring_gen_z8_source_ids s WHERE s.id = m.id
+ )::INT[] AS source_ids, m.geometry, m.highway, m.network, m.construction,
+ visible_brunnel(m.geometry, m.is_bridge, 9) AS is_bridge,
+ visible_brunnel(m.geometry, m.is_tunnel, 9) AS is_tunnel,
+ visible_brunnel(m.geometry, m.is_ford, 9) AS is_ford, m.expressway, m.z_order
+ FROM linestrings_to_merge
+ JOIN osm_transportation_merge_linestring_gen_z8 m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry));
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z8 m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_transportation_merge_linestring_gen_z8_source_ids m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
+ ) as cluster_group
+ FROM linestrings_to_merge;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
+
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
+
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_transportation_merge_linestring_gen_z8 ADD COLUMN IF NOT EXISTS new_source_ids INT[];
+ ALTER TABLE osm_transportation_merge_linestring_gen_z8 ADD COLUMN IF NOT EXISTS old_source_ids INT[];
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_transportation_merge_linestring_gen_z8(geometry, new_source_ids, old_source_ids, highway,
+ network, construction, is_bridge, is_tunnel, is_ford,
+ expressway, z_order)
+ SELECT (ST_Dump(ST_Simplify(ST_LineMerge(ST_Union(geometry)), ZRes(10)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries to
+ -- merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+ -- more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(source_id) FILTER (WHERE source_id IS NOT NULL), '{}' )::INT[] AS new_source_ids,
+ array_cat_agg(source_ids)::INT[] as old_source_ids,
+ highway,
+ network,
+ construction,
+ is_bridge,
+ is_tunnel,
+ is_ford,
+ expressway,
+ min(z_order) as z_order
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway
+ RETURNING id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources. This required because
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+ -- more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_transportation_merge_linestring_gen_z8_source_ids (id, source_id)
+ SELECT m.id, m.source_id
+ FROM (
+ SELECT id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_transportation_merge_linestring_gen_z9 s ON (m.source_id = s.id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (id, source_id) DO NOTHING;
+
+ -- Cleanup
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_transportation_merge_linestring_gen_z8 DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_transportation_merge_linestring_gen_z8 DROP COLUMN IF EXISTS old_source_ids;
- DROP TABLE osm_highway_linestring_original;
- DROP TABLE changes_compact;
-- noinspection SqlWithoutWhere
DELETE FROM transportation.changes_z9;
-- noinspection SqlWithoutWhere
DELETE FROM transportation.updates_z9;
- RAISE LOG 'Refresh transportation z9 done in %', age(clock_timestamp(), t);
+ RAISE LOG 'Refresh transportation z8 done in %', age(clock_timestamp(), t);
+
+ -- Update z7, z6, z5 and z4 tables
+ PERFORM insert_transportation_merge_linestring_gen_z7(FALSE);
+
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
-
CREATE TRIGGER trigger_store_transportation_highway_linestring_gen_z9
AFTER INSERT OR UPDATE OR DELETE
ON osm_transportation_merge_linestring_gen_z9
FOR EACH ROW
EXECUTE PROCEDURE transportation.store_z9();
+CREATE TRIGGER trigger_store_osm_transportation_merge_linestring_gen_z8
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_transportation_merge_linestring_gen_z8
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation.store_z8();
+
CREATE TRIGGER trigger_flag_transportation_z9
AFTER INSERT OR UPDATE OR DELETE
ON osm_transportation_merge_linestring_gen_z9
@@ -843,35 +1438,3 @@ CREATE CONSTRAINT TRIGGER trigger_refresh_z8
INITIALLY DEFERRED
FOR EACH ROW
EXECUTE PROCEDURE transportation.refresh_z8();
-
-
--- Handle updates on
--- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z7
--- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z6
--- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z5
--- osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z4
-
-
-CREATE OR REPLACE FUNCTION transportation.merge_linestring_gen_refresh_z7() RETURNS trigger AS
-$$
-BEGIN
- IF (tg_op = 'DELETE') THEN
- DELETE FROM osm_transportation_merge_linestring_gen_z7 WHERE id = old.id;
- DELETE FROM osm_transportation_merge_linestring_gen_z6 WHERE id = old.id;
- DELETE FROM osm_transportation_merge_linestring_gen_z5 WHERE id = old.id;
- DELETE FROM osm_transportation_merge_linestring_gen_z4 WHERE id = old.id;
- END IF;
-
- IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
- PERFORM insert_transportation_merge_linestring_gen_z7(new.id);
- END IF;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER trigger_osm_transportation_merge_linestring_gen_z8
- AFTER INSERT OR UPDATE OR DELETE
- ON osm_transportation_merge_linestring_gen_z8
- FOR EACH ROW
-EXECUTE PROCEDURE transportation.merge_linestring_gen_refresh_z7();
diff --git a/layers/transportation_name/etl_diagram.png b/layers/transportation_name/etl_diagram.png
index d46ad20c..c4ee4e00 100644
Binary files a/layers/transportation_name/etl_diagram.png and b/layers/transportation_name/etl_diagram.png differ
diff --git a/layers/transportation_name/highway_classification.sql b/layers/transportation_name/highway_classification.sql
index b9f8e07e..b7d5e7a4 100644
--- a/layers/transportation_name/highway_classification.sql
+++ b/layers/transportation_name/highway_classification.sql
@@ -1,52 +1,46 @@
CREATE OR REPLACE FUNCTION highway_to_val(hwy_class varchar)
RETURNS int
IMMUTABLE
-LANGUAGE plpgsql
+LANGUAGE sql
AS $$
-BEGIN
- CASE hwy_class
- WHEN 'motorway' THEN RETURN 6;
- WHEN 'trunk' THEN RETURN 5;
- WHEN 'primary' THEN RETURN 4;
- WHEN 'secondary' THEN RETURN 3;
- WHEN 'tertiary' THEN RETURN 2;
- WHEN 'unclassified' THEN RETURN 1;
- else RETURN 0;
- END CASE;
-END;
+ SELECT CASE hwy_class
+ WHEN 'motorway' THEN 6
+ WHEN 'trunk' THEN 5
+ WHEN 'primary' THEN 4
+ WHEN 'secondary' THEN 3
+ WHEN 'tertiary' THEN 2
+ WHEN 'unclassified' THEN 1
+ ELSE 0
+ END;
$$;
CREATE OR REPLACE FUNCTION val_to_highway(hwy_val int)
RETURNS varchar
IMMUTABLE
-LANGUAGE plpgsql
+LANGUAGE sql
AS $$
-BEGIN
- CASE hwy_val
- WHEN 6 THEN RETURN 'motorway';
- WHEN 5 THEN RETURN 'trunk';
- WHEN 4 THEN RETURN 'primary';
- WHEN 3 THEN RETURN 'secondary';
- WHEN 2 THEN RETURN 'tertiary';
- WHEN 1 THEN RETURN 'unclassified';
- else RETURN null;
- END CASE;
-END;
+ SELECT CASE hwy_val
+ WHEN 6 THEN 'motorway'
+ WHEN 5 THEN 'trunk'
+ WHEN 4 THEN 'primary'
+ WHEN 3 THEN 'secondary'
+ WHEN 2 THEN 'tertiary'
+ WHEN 1 THEN 'unclassified'
+ ELSE null
+ END;
$$;
CREATE OR REPLACE FUNCTION highest_hwy_sfunc(agg_state varchar, hwy_class varchar)
RETURNS varchar
IMMUTABLE
-LANGUAGE plpgsql
+LANGUAGE sql
AS $$
-BEGIN
- RETURN val_to_highway(
+ SELECT val_to_highway(
GREATEST(
highway_to_val(agg_state),
highway_to_val(hwy_class)
)
);
-END;
$$;
DROP AGGREGATE IF EXISTS highest_highway (varchar);
diff --git a/layers/transportation_name/transportation_name.sql b/layers/transportation_name/transportation_name.sql
index 60d0fd8b..db0e9010 100644
--- a/layers/transportation_name/transportation_name.sql
+++ b/layers/transportation_name/transportation_name.sql
@@ -54,7 +54,20 @@ SELECT geometry,
FROM (
-- etldoc: osm_transportation_name_linestring_gen4 -> layer_transportation_name:z6
- SELECT *,
+ SELECT geometry,
+ tags,
+ ref,
+ highway,
+ subclass,
+ brunnel,
+ network,
+ route_1,
+ route_2,
+ route_3,
+ route_4,
+ route_5,
+ route_6,
+ z_order,
NULL::int AS layer,
NULL::int AS level,
NULL::boolean AS indoor
@@ -63,32 +76,71 @@ FROM (
UNION ALL
-- etldoc: osm_transportation_name_linestring_gen3 -> layer_transportation_name:z7
- SELECT *,
+ SELECT geometry,
+ tags,
+ ref,
+ highway,
+ subclass,
+ brunnel,
+ network,
+ route_1,
+ route_2,
+ route_3,
+ route_4,
+ route_5,
+ route_6,
+ z_order,
NULL::int AS layer,
NULL::int AS level,
NULL::boolean AS indoor
FROM osm_transportation_name_linestring_gen3
- WHERE zoom_level = 7
+ WHERE ST_Length(geometry) > 20000 AND zoom_level = 7
UNION ALL
-- etldoc: osm_transportation_name_linestring_gen2 -> layer_transportation_name:z8
- SELECT *,
+ SELECT geometry,
+ tags,
+ ref,
+ highway,
+ subclass,
+ brunnel,
+ network,
+ route_1,
+ route_2,
+ route_3,
+ route_4,
+ route_5,
+ route_6,
+ z_order,
NULL::int AS layer,
NULL::int AS level,
NULL::boolean AS indoor
FROM osm_transportation_name_linestring_gen2
- WHERE zoom_level = 8
+ WHERE ST_Length(geometry) > 14000 AND zoom_level = 8
UNION ALL
-- etldoc: osm_transportation_name_linestring_gen1 -> layer_transportation_name:z9
-- etldoc: osm_transportation_name_linestring_gen1 -> layer_transportation_name:z10
-- etldoc: osm_transportation_name_linestring_gen1 -> layer_transportation_name:z11
- SELECT *,
+ SELECT geometry,
+ tags,
+ ref,
+ highway,
+ subclass,
+ brunnel,
+ network,
+ route_1,
+ route_2,
+ route_3,
+ route_4,
+ route_5,
+ route_6,
+ z_order,
NULL::int AS layer,
NULL::int AS level,
NULL::boolean AS indoor
FROM osm_transportation_name_linestring_gen1
- WHERE zoom_level BETWEEN 9 AND 11
+ WHERE ST_Length(geometry) > 8000 / POWER(2, zoom_level - 9) AND zoom_level BETWEEN 9 AND 11
UNION ALL
-- etldoc: osm_transportation_name_linestring -> layer_transportation_name:z12
diff --git a/layers/transportation_name/update_transportation_name.sql b/layers/transportation_name/update_transportation_name.sql
index d04d1495..e6496d11 100644
--- a/layers/transportation_name/update_transportation_name.sql
+++ b/layers/transportation_name/update_transportation_name.sql
@@ -1,13 +1,86 @@
+DROP TRIGGER IF EXISTS trigger_store_transportation_route_member ON osm_route_member;
+DROP TRIGGER IF EXISTS trigger_store_transportation_superroute_member ON osm_superroute_member;
+DROP TRIGGER IF EXISTS trigger_store_transportation_highway_linestring ON osm_highway_linestring;
+DROP TRIGGER IF EXISTS trigger_flag_transportation_name ON transportation_name.network_changes;
+DROP TRIGGER IF EXISTS trigger_refresh_network ON transportation_name.updates_network;
+DROP TRIGGER IF EXISTS trigger_store_transportation_name_network ON osm_transportation_name_network;
+DROP TRIGGER IF EXISTS trigger_store_transportation_name_shipway ON osm_shipway_linestring;
+DROP TRIGGER IF EXISTS trigger_store_transportation_name_aerialway ON osm_aerialway_linestring;
+DROP TRIGGER IF EXISTS trigger_store_transportation_name_linestring ON osm_transportation_name_linestring;
+DROP TRIGGER IF EXISTS trigger_flag_name ON transportation_name.name_changes;
+DROP TRIGGER IF EXISTS trigger_flag_shipway ON transportation_name.shipway_changes;
+DROP TRIGGER IF EXISTS trigger_flag_aerialway ON transportation_name.aerialway_changes;
+DROP TRIGGER IF EXISTS trigger_refresh_name ON transportation_name.updates_name;
+DROP TRIGGER IF EXISTS trigger_refresh_shipway ON transportation_name.updates_shipway;
+DROP TRIGGER IF EXISTS trigger_refresh_aerialway ON transportation_name.updates_aerialway;
+
-- Instead of using relations to find out the road names we
-- stitch together the touching ways with the same name
-- to allow for nice label rendering
-- Because this works well for roads that do not have relations as well
+-- Indexes for filling and updating osm_transportation_name_linestring table
+CREATE UNIQUE INDEX IF NOT EXISTS osm_shipway_linestring_update_partial_idx ON osm_shipway_linestring (osm_id)
+ WHERE name <> '';
+CREATE UNIQUE INDEX IF NOT EXISTS osm_aerialway_linestring_update_partial_idx ON osm_aerialway_linestring (osm_id)
+ WHERE name <> '';
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_network_update_partial_idx
+ ON osm_transportation_name_network (osm_id)
+ WHERE coalesce(tags->'name', '') <> '' OR
+ coalesce(ref, '') <> '';
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_network_osm_id_idx ON osm_transportation_name_network (osm_id);
+
+-- Analyze tables with indexes created on them
+ANALYZE osm_aerialway_linestring, osm_shipway_linestring, osm_transportation_name_network;
+
-- etldoc: osm_transportation_name_network -> osm_transportation_name_linestring
-- etldoc: osm_shipway_linestring -> osm_transportation_name_linestring
-- etldoc: osm_aerialway_linestring -> osm_transportation_name_linestring
-CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring AS
-SELECT (ST_Dump(geometry)).geom AS geometry,
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring(
+ id SERIAL,
+ source integer,
+ geometry geometry('LineString'),
+ source_ids bigint[],
+ tags hstore,
+ ref text,
+ highway varchar,
+ subclass text,
+ brunnel text,
+ sac_scale varchar,
+ "level" integer,
+ layer integer,
+ indoor boolean,
+ network route_network_type,
+ route_1 text,
+ route_2 text,
+ route_3 text,
+ route_4 text,
+ route_5 text,
+ route_6 text,
+ z_order integer,
+ route_rank integer
+);
+
+-- Create OneToMany-Relation-Table storing relations of a Merged-LineString in table
+-- osm_transportation_name_linestring to Source-LineStrings from tables osm_transportation_name_network,
+-- osm_shipway_linestring and osm_aerialway_linestring
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_source_ids(
+ source int,
+ id int,
+ source_id bigint,
+ PRIMARY KEY (source, id, source_id)
+);
+
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_name_linestring;
+TRUNCATE osm_transportation_name_linestring_source_ids;
+
+INSERT INTO osm_transportation_name_linestring(source, geometry, source_ids, tags, ref, highway, subclass, brunnel,
+ sac_scale, "level", layer, indoor, network, route_1, route_2,
+ route_3, route_4, route_5, route_6,z_order, route_rank)
+SELECT source,
+ geometry,
+ source_ids,
tags || get_basic_names(tags, geometry) AS tags,
ref,
highway,
@@ -22,30 +95,70 @@ SELECT (ST_Dump(geometry)).geom AS geometry,
z_order,
route_rank
FROM (
- SELECT ST_LineMerge(ST_Collect(geometry)) AS geometry,
+ -- Merge LineStrings from osm_transportation_name_network by grouping them and creating intersecting
+ -- clusters of each group via ST_ClusterDBSCAN
+ SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the
+ -- geometries to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be
+ -- found. https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(osm_id) AS source_ids,
+ 0 AS source,
tags,
ref,
highway,
subclass,
- CASE WHEN COUNT(*) = COUNT(brunnel) AND MAX(brunnel) = MIN(brunnel)
- THEN MAX(brunnel)
- ELSE NULL::text END AS brunnel,
+ brunnel,
sac_scale,
- "level",
+ level,
layer,
indoor,
network_type,
route_1, route_2, route_3, route_4, route_5, route_6,
min(z_order) AS z_order,
min(route_rank) AS route_rank
- FROM osm_transportation_name_network
- WHERE tags->'name' <> '' OR ref <> ''
- GROUP BY tags, ref, highway, subclass, "level", layer, sac_scale, indoor, network_type,
- route_1, route_2, route_3, route_4, route_5, route_6
+ FROM (
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points
+ -- to 1. https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor,
+ network_type, route_1, route_2, route_3, route_4, route_5, route_6
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition
+ -- starting at 0. This leads to clusters having the same ID across multiple partitions
+ -- therefore we generate a Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the
+ -- partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor,
+ network_type, route_1, route_2, route_3, route_4, route_5, route_6
+ ) as cluster_group
+ FROM osm_transportation_name_network
+ WHERE coalesce(tags->'name', '') <> '' OR
+ coalesce(ref, '') <> ''
+ ) q
+ GROUP BY cluster_group, cluster, tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor,
+ network_type, route_1, route_2, route_3, route_4, route_5, route_6
UNION ALL
- SELECT ST_LineMerge(ST_Collect(geometry)) AS geometry,
- transportation_name_tags(NULL::geometry, tags, name, name_en, name_de) AS tags,
+ -- Merge LineStrings from osm_shipway_linestring by grouping them and creating intersecting
+ -- clusters of each group via ST_ClusterDBSCAN
+ SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the
+ -- geometries to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be
+ -- found. https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(osm_id) AS source_ids,
+ 1 AS source,
+ transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ) AS tags,
NULL AS ref,
'shipway' AS highway,
shipway AS subclass,
@@ -63,13 +176,47 @@ FROM (
NULL AS route_6,
min(z_order) AS z_order,
NULL::int AS route_rank
- FROM osm_shipway_linestring
- WHERE name <> ''
- GROUP BY name, name_en, name_de, tags, subclass, "level", layer
+ FROM (
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points
+ -- to 1. https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), shipway, layer
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition
+ -- starting at 0. This leads to clusters having the same ID across multiple partitions
+ -- therefore we generate a Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the
+ -- partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), shipway, layer
+ ) as cluster_group
+ FROM osm_shipway_linestring
+ WHERE name <> ''
+ ) q
+ GROUP BY cluster_group, cluster, transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), shipway, layer
UNION ALL
- SELECT ST_LineMerge(ST_Collect(geometry)) AS geometry,
- transportation_name_tags(NULL::geometry, tags, name, name_en, name_de) AS tags,
+ -- Merge LineStrings from osm_aerialway_linestring by grouping them and creating intersecting
+ -- clusters of each group via ST_ClusterDBSCAN
+ SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the
+ -- geometries to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be
+ -- found. https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(osm_id) AS source_ids,
+ 2 AS source,
+ transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ) AS tags,
NULL AS ref,
'aerialway' AS highway,
aerialway AS subclass,
@@ -87,144 +234,431 @@ FROM (
NULL AS route_6,
min(z_order) AS z_order,
NULL::int AS route_rank
- FROM osm_aerialway_linestring
- WHERE name <> ''
- GROUP BY name, name_en, name_de, tags, subclass, "level", layer
- ) AS highway_union
-;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_name_ref_idx ON osm_transportation_name_linestring (coalesce(tags->'name', ''), coalesce(ref, ''));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx ON osm_transportation_name_linestring USING gist (geometry);
+ FROM (
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points
+ -- to 1. https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), aerialway, layer
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition
+ -- starting at 0. This leads to clusters having the same ID across multiple partitions
+ -- therefore we generate a Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the
+ -- partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), aerialway, layer
+ ) as cluster_group
+ FROM osm_aerialway_linestring
+ WHERE name <> ''
+ ) q
+ GROUP BY cluster_group, cluster, transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ), aerialway, layer
+ ) AS highway_union;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_highway_partial_idx
- ON osm_transportation_name_linestring (highway, subclass)
- WHERE highway IN ('motorway', 'trunk', 'construction');
+-- Geometry Index
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx
+ ON osm_transportation_name_linestring USING gist (geometry);
--- etldoc: osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1
-CREATE OR REPLACE VIEW osm_transportation_name_linestring_gen1_view AS
-SELECT ST_Simplify(geometry, 50) AS geometry,
- tags,
- ref,
- highway,
- subclass,
- brunnel,
- network,
- route_1, route_2, route_3, route_4, route_5, route_6,
- z_order
-FROM osm_transportation_name_linestring
-WHERE (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
- AND ST_Length(geometry) > 8000
-;
-CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen1 AS
-SELECT *
-FROM osm_transportation_name_linestring_gen1_view;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_name_ref_idx ON osm_transportation_name_linestring_gen1((coalesce(tags->'name', ref)));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx ON osm_transportation_name_linestring_gen1 USING gist (geometry);
+-- Create table for simplified LineStrings
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen1 (
+ id integer,
+ geometry geometry,
+ tags hstore,
+ ref text,
+ highway varchar,
+ subclass text,
+ brunnel text,
+ network route_network_type,
+ route_1 text,
+ route_2 text,
+ route_3 text,
+ route_4 text,
+ route_5 text,
+ route_6 text,
+ z_order integer
+);
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_highway_partial_idx
- ON osm_transportation_name_linestring_gen1 (highway, subclass)
- WHERE highway IN ('motorway', 'trunk', 'construction');
+-- Create osm_transportation_name_linestring_gen2 as a copy of osm_transportation_name_linestring_gen1
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen2
+(LIKE osm_transportation_name_linestring_gen1);
--- etldoc: osm_transportation_name_linestring_gen1 -> osm_transportation_name_linestring_gen2
-CREATE OR REPLACE VIEW osm_transportation_name_linestring_gen2_view AS
-SELECT ST_Simplify(geometry, 120) AS geometry,
- tags,
- ref,
- highway,
- subclass,
- brunnel,
- network,
- route_1, route_2, route_3, route_4, route_5, route_6,
- z_order
-FROM osm_transportation_name_linestring_gen1
-WHERE (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
- AND ST_Length(geometry) > 14000
-;
-CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen2 AS
-SELECT *
-FROM osm_transportation_name_linestring_gen2_view;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_name_ref_idx ON osm_transportation_name_linestring_gen2((coalesce(tags->'name', ref)));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx ON osm_transportation_name_linestring_gen2 USING gist (geometry);
+-- Create osm_transportation_name_linestring_gen3 as a copy of osm_transportation_name_linestring_gen2
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen3
+(LIKE osm_transportation_name_linestring_gen2);
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_highway_partial_idx
- ON osm_transportation_name_linestring_gen2 (highway, subclass)
- WHERE highway IN ('motorway', 'trunk', 'construction');
+-- Create osm_transportation_name_linestring_gen4 as a copy of osm_transportation_name_linestring_gen3
+CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen4
+(LIKE osm_transportation_name_linestring_gen3);
--- etldoc: osm_transportation_name_linestring_gen2 -> osm_transportation_name_linestring_gen3
-CREATE OR REPLACE VIEW osm_transportation_name_linestring_gen3_view AS
-SELECT ST_Simplify(geometry, 200) AS geometry,
- tags,
- ref,
- highway,
- subclass,
- brunnel,
- network,
- route_1, route_2, route_3, route_4, route_5, route_6,
- z_order
-FROM osm_transportation_name_linestring_gen2
-WHERE (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway')
- AND ST_Length(geometry) > 20000
-;
-CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen3 AS
-SELECT *
-FROM osm_transportation_name_linestring_gen3_view;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_name_ref_idx ON osm_transportation_name_linestring_gen3((coalesce(tags->'name', ref)));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist (geometry);
+-- Create Primary-Keys for osm_transportation_name_linestring and
+-- osm_transportation_name_linestring_gen1/gen2/gen3/gen4 tables
+DO $$
+BEGIN
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_linestring' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_linestring ADD PRIMARY KEY (id);
+ END IF;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_highway_partial_idx
- ON osm_transportation_name_linestring_gen3 (highway, subclass)
- WHERE highway IN ('motorway', 'construction');
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_linestring_gen1' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_linestring_gen1 ADD PRIMARY KEY (id);
+ END IF;
--- etldoc: osm_transportation_name_linestring_gen3 -> osm_transportation_name_linestring_gen4
-CREATE OR REPLACE VIEW osm_transportation_name_linestring_gen4_view AS
-SELECT ST_Simplify(geometry, 500) AS geometry,
- tags,
- ref,
- highway,
- subclass,
- brunnel,
- network,
- route_1, route_2, route_3, route_4, route_5, route_6,
- z_order
-FROM osm_transportation_name_linestring_gen3
-WHERE (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway')
- AND ST_Length(geometry) > 20000
-;
-CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen4 AS
-SELECT *
-FROM osm_transportation_name_linestring_gen4_view;
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen4_name_ref_idx ON osm_transportation_name_linestring_gen4((coalesce(tags->'name', ref)));
-CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen4_geometry_idx ON osm_transportation_name_linestring_gen4 USING gist (geometry);
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_linestring_gen2' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_linestring_gen2 ADD PRIMARY KEY (id);
+ END IF;
--- Handle updates
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_linestring_gen3' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_linestring_gen3 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_transportation_name_linestring_gen4' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_transportation_name_linestring_gen4 ADD PRIMARY KEY (id);
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Indexes which can be utilized during full-update for queries originating from
+-- update_transportation_name_linestring_gen() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_linestring_update_partial_idx
+ ON osm_transportation_name_linestring (id)
+ WHERE (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
+ AND ST_Length(geometry) > 8000;
+
+-- Temporary index for filling source tables
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_source_idx ON osm_transportation_name_linestring (source);
+
+-- Analyze populated table with indexes
+ANALYZE osm_transportation_name_linestring;
+
+-- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources. This required because
+-- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+-- more than two LineStrings form an intersection or no intersection could be found.
+-- Execute after indexes have been created on osm_transportation_merge_linestring_gen_z11 to improve performance
+INSERT INTO osm_transportation_name_linestring_source_ids(source, id, source_id)
+SELECT m.source, m.id, source_id
+FROM (
+ SELECT id, source, unnest(source_ids) AS source_id, geometry
+ FROM osm_transportation_name_linestring
+ WHERE osm_transportation_name_linestring.source = 0
+) m
+JOIN osm_transportation_name_network s ON (m.source_id = s.osm_id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (source, id, source_id) DO NOTHING;
+INSERT INTO osm_transportation_name_linestring_source_ids(source, id, source_id)
+SELECT m.source, m.id, source_id
+FROM (
+ SELECT id, source, unnest(source_ids) AS source_id, geometry
+ FROM osm_transportation_name_linestring
+ WHERE osm_transportation_name_linestring.source = 1
+) m
+JOIN osm_shipway_linestring s ON (m.source_id = s.osm_id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (source, id, source_id) DO NOTHING;
+INSERT INTO osm_transportation_name_linestring_source_ids(source, id, source_id)
+SELECT m.source, m.id, source_id
+FROM (
+ SELECT id, source, unnest(source_ids) AS source_id, geometry
+ FROM osm_transportation_name_linestring
+ WHERE osm_transportation_name_linestring.source = 2
+) m
+JOIN osm_aerialway_linestring s ON (m.source_id = s.osm_id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (source, id, source_id) DO NOTHING;
+
+-- Drop temporary Merged-LineString to Source-LineStrings-ID column
+ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS source_ids;
+
+-- Drop temporary index
+DROP INDEX IF EXISTS osm_transportation_name_linestring_source_idx;
CREATE SCHEMA IF NOT EXISTS transportation_name;
+CREATE TABLE IF NOT EXISTS transportation_name.name_changes_gen
+(
+ is_old boolean,
+ id int,
+ PRIMARY KEY (is_old, id)
+);
+
+CREATE OR REPLACE FUNCTION update_transportation_name_linestring_gen (full_update bool) RETURNS VOID AS $$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh transportation_name merged';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation_name.name_changes_gen;
+ ANALYZE osm_transportation_name_linestring;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_transportation_name_linestring_gen1
+ USING transportation_name.name_changes_gen
+ WHERE full_update IS TRUE OR (
+ transportation_name.name_changes_gen.is_old IS TRUE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen1.id
+ );
+
+ -- etldoc: osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1
+ INSERT INTO osm_transportation_name_linestring_gen1 (id, geometry, tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order)
+ SELECT MIN(id) as id,
+ ST_Simplify(ST_LineMerge(ST_Collect(geometry)), 50) AS geometry,
+ tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM (
+ SELECT id,
+ geometry,
+ tags, ref, highway, subclass,
+ CASE WHEN ST_Length(geometry) > 8000 THEN brunnel ELSE '' END AS brunnel,
+ network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM osm_transportation_name_linestring
+ ) osm_transportation_name_linestring_gen1_pre_merge
+ WHERE (
+ full_update IS TRUE OR EXISTS (
+ SELECT NULL
+ FROM transportation_name.name_changes_gen
+ WHERE transportation_name.name_changes_gen.is_old IS FALSE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen1_pre_merge.id
+ )
+ ) AND (
+ (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
+ )
+ GROUP BY tags, ref, highway, subclass, brunnel, network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, tags = excluded.tags, ref = excluded.ref,
+ highway = excluded.highway, subclass = excluded.subclass,
+ brunnel = excluded.brunnel, network = excluded.network, route_1 = excluded.route_1,
+ route_2 = excluded.route_2, route_3 = excluded.route_3, route_4 = excluded.route_4,
+ route_5 = excluded.route_5, route_6 = excluded.route_6, z_order = excluded.z_order;
+
+ -- Analyze source table
+ ANALYZE osm_transportation_name_linestring_gen1;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_transportation_name_linestring_gen2
+ USING transportation_name.name_changes_gen
+ WHERE full_update IS TRUE OR (
+ transportation_name.name_changes_gen.is_old IS TRUE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen2.id
+ );
+
+ -- etldoc: osm_transportation_name_linestring_gen1 -> osm_transportation_name_linestring_gen2
+ INSERT INTO osm_transportation_name_linestring_gen2 (id, geometry, tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order)
+ SELECT MIN(id) as id,
+ ST_Simplify(ST_LineMerge(ST_Collect(geometry)), 120) AS geometry,
+ tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM (
+ SELECT id,
+ (ST_Dump(geometry)).geom AS geometry,
+ tags, ref, highway, subclass,
+ CASE WHEN ST_Length(geometry) > 14000 THEN brunnel ELSE '' END AS brunnel,
+ network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM osm_transportation_name_linestring_gen1
+ ) osm_transportation_name_linestring_gen2_pre_merge
+ WHERE (
+ full_update IS TRUE OR EXISTS (
+ SELECT NULL
+ FROM transportation_name.name_changes_gen
+ WHERE transportation_name.name_changes_gen.is_old IS FALSE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen2_pre_merge.id
+ )
+ ) AND (
+ (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
+ )
+ GROUP BY tags, ref, highway, subclass, brunnel, network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, tags = excluded.tags, ref = excluded.ref,
+ highway = excluded.highway, subclass = excluded.subclass,
+ brunnel = excluded.brunnel, network = excluded.network, route_1 = excluded.route_1,
+ route_2 = excluded.route_2, route_3 = excluded.route_3, route_4 = excluded.route_4,
+ route_5 = excluded.route_5, route_6 = excluded.route_6, z_order = excluded.z_order;
+
+ -- Analyze source table
+ ANALYZE osm_transportation_name_linestring_gen2;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_transportation_name_linestring_gen3
+ USING transportation_name.name_changes_gen
+ WHERE full_update IS TRUE OR (
+ transportation_name.name_changes_gen.is_old IS TRUE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen3.id
+ );
+
+ -- etldoc: osm_transportation_name_linestring_gen2 -> osm_transportation_name_linestring_gen3
+ INSERT INTO osm_transportation_name_linestring_gen3 (id, geometry, tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order)
+ SELECT MIN(id) as id,
+ ST_Simplify(ST_LineMerge(ST_Collect(geometry)), 200) AS geometry,
+ tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM (
+ SELECT id,
+ (ST_Dump(geometry)).geom AS geometry,
+ tags, ref, highway, subclass,
+ CASE WHEN ST_Length(geometry) > 20000 THEN brunnel ELSE '' END AS brunnel,
+ network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ FROM osm_transportation_name_linestring_gen2
+ ) osm_transportation_name_linestring_gen3_pre_merge
+ WHERE (
+ full_update IS TRUE OR EXISTS (
+ SELECT NULL
+ FROM transportation_name.name_changes_gen
+ WHERE transportation_name.name_changes_gen.is_old IS FALSE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen3_pre_merge.id
+ )
+ ) AND (
+ (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway')
+ )
+ GROUP BY tags, ref, highway, subclass, brunnel, network, route_1, route_2, route_3, route_4, route_5, route_6, z_order
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, tags = excluded.tags, ref = excluded.ref,
+ highway = excluded.highway, subclass = excluded.subclass,
+ brunnel = excluded.brunnel, network = excluded.network, route_1 = excluded.route_1,
+ route_2 = excluded.route_2, route_3 = excluded.route_3, route_4 = excluded.route_4,
+ route_5 = excluded.route_5, route_6 = excluded.route_6, z_order = excluded.z_order;
+
+ -- Analyze source table
+ ANALYZE osm_transportation_name_linestring_gen3;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_transportation_name_linestring_gen4
+ USING transportation_name.name_changes_gen
+ WHERE full_update IS TRUE OR (
+ transportation_name.name_changes_gen.is_old IS TRUE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen4.id
+ );
+
+ -- etldoc: osm_transportation_name_linestring_gen3 -> osm_transportation_name_linestring_gen4
+ INSERT INTO osm_transportation_name_linestring_gen4 (id, geometry, tags, ref, highway, subclass, brunnel, network,
+ route_1, route_2, route_3, route_4, route_5, route_6, z_order)
+ SELECT id, ST_Simplify(geometry, 500) AS geometry, tags, ref, highway, subclass, brunnel, network, route_1, route_2,
+ route_3, route_4, route_5, route_6, z_order
+ FROM osm_transportation_name_linestring_gen3
+ WHERE (
+ full_update IS TRUE OR EXISTS (
+ SELECT NULL
+ FROM transportation_name.name_changes_gen
+ WHERE transportation_name.name_changes_gen.is_old IS FALSE AND
+ transportation_name.name_changes_gen.id = osm_transportation_name_linestring_gen3.id
+ )
+ ) AND (
+ (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway') AND
+ ST_Length(geometry) > 20000
+ ) ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, tags = excluded.tags, ref = excluded.ref,
+ highway = excluded.highway, subclass = excluded.subclass,
+ brunnel = excluded.brunnel, network = excluded.network, route_1 = excluded.route_1,
+ route_2 = excluded.route_2, route_3 = excluded.route_3, route_4 = excluded.route_4,
+ route_5 = excluded.route_5, route_6 = excluded.route_6, z_order = excluded.z_order;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation_name.name_changes_gen;
+
+ RAISE LOG 'Refresh transportation_name merged done in %', age(clock_timestamp(), t);
+END;
+$$ LANGUAGE plpgsql;
+
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_transportation_name_linestring_gen1;
+TRUNCATE osm_transportation_name_linestring_gen2;
+TRUNCATE osm_transportation_name_linestring_gen3;
+TRUNCATE osm_transportation_name_linestring_gen4;
+
+SELECT update_transportation_name_linestring_gen(TRUE);
+
+-- Indexes for queries originating from update_transportation_name_linestring_gen() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_update_partial_idx
+ ON osm_transportation_name_linestring_gen1 (id)
+ WHERE (highway IN ('motorway', 'trunk') OR highway = 'construction' AND subclass IN ('motorway', 'trunk'))
+ AND ST_Length(geometry) > 14000;
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_update_partial_idx
+ ON osm_transportation_name_linestring_gen2 (id)
+ WHERE (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway')
+ AND ST_Length(geometry) > 20000;
+CREATE UNIQUE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_update_partial_idx
+ ON osm_transportation_name_linestring_gen3 (id)
+ WHERE (highway = 'motorway' OR highway = 'construction' AND subclass = 'motorway')
+ AND ST_Length(geometry) > 20000;
+
+-- Geometry Indexes
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx
+ ON osm_transportation_name_linestring_gen1 USING gist (geometry);
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx
+ ON osm_transportation_name_linestring_gen2 USING gist (geometry);
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx
+ ON osm_transportation_name_linestring_gen3 USING gist (geometry);
+CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen4_geometry_idx
+ ON osm_transportation_name_linestring_gen4 USING gist (geometry);
+
+-- Handle updates
+
-- Trigger to update "osm_transportation_name_network" from "osm_route_member" and "osm_highway_linestring"
CREATE TABLE IF NOT EXISTS transportation_name.network_changes
(
+ is_old bool,
osm_id bigint,
- UNIQUE (osm_id)
+ PRIMARY KEY (is_old, osm_id)
);
+-- Store IDs of changed elements from osm_route_member table.
CREATE OR REPLACE FUNCTION transportation_name.route_member_store() RETURNS trigger AS
$$
BEGIN
- INSERT INTO transportation_name.network_changes(osm_id)
- VALUES (CASE WHEN tg_op IN ('DELETE', 'UPDATE') THEN old.member ELSE new.member END)
- ON CONFLICT(osm_id) DO NOTHING;
-
+ IF tg_op = 'DELETE' OR (tg_op = 'UPDATE' AND (old.member IS DISTINCT FROM new.member))
+ THEN
+ INSERT INTO transportation_name.network_changes(is_old, osm_id)
+ VALUES (TRUE, old.member)
+ ON CONFLICT(is_old, osm_id) DO NOTHING;
+ END IF;
+ IF (tg_op IN ('UPDATE', 'INSERT'))
+ THEN
+ INSERT INTO transportation_name.network_changes(is_old, osm_id)
+ VALUES (FALSE, new.member)
+ ON CONFLICT(is_old, osm_id) DO NOTHING;
+ END IF;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
+-- Store IDs of changed elements from osm_highway_linestring table.
CREATE OR REPLACE FUNCTION transportation_name.highway_linestring_store() RETURNS trigger AS
$$
BEGIN
- INSERT INTO transportation_name.network_changes(osm_id)
- VALUES (CASE WHEN tg_op IN ('DELETE', 'UPDATE') THEN old.osm_id ELSE new.osm_id END)
- ON CONFLICT(osm_id) DO NOTHING;
-
+ IF tg_op = 'DELETE' OR (tg_op = 'UPDATE' AND (old.osm_id IS DISTINCT FROM new.osm_id))
+ THEN
+ INSERT INTO transportation_name.network_changes(is_old, osm_id)
+ VALUES (TRUE, old.osm_id)
+ ON CONFLICT(is_old, osm_id) DO NOTHING;
+ END IF;
+ IF (tg_op IN ('UPDATE', 'INSERT'))
+ THEN
+ INSERT INTO transportation_name.network_changes(is_old, osm_id)
+ VALUES (FALSE, new.osm_id)
+ ON CONFLICT(is_old, osm_id) DO NOTHING;
+ END IF;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -249,30 +683,38 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh transportation_name_network';
- PERFORM update_osm_route_member();
+
+ -- Update Way-Relations and analyze table afterwards
+ PERFORM update_osm_route_member(FALSE);
+ ANALYZE transportation_route_member_coalesced;
-- REFRESH osm_transportation_name_network
DELETE
- FROM osm_transportation_name_network AS n
- USING
- transportation_name.network_changes AS c
- WHERE n.osm_id = c.osm_id;
+ FROM osm_transportation_name_network
+ USING transportation_name.network_changes c
+ WHERE c.is_old IS TRUE AND osm_transportation_name_network.osm_id = c.osm_id;
- UPDATE osm_highway_linestring hl
- SET network = rm.network_type
- FROM transportation_name.network_changes c,
- osm_route_member rm
- WHERE hl.osm_id=c.osm_id
- AND hl.osm_id=rm.member
- AND rm.concurrency_index=1;
+ UPDATE osm_highway_linestring
+ SET network = NULL
+ FROM transportation_name.network_changes c
+ WHERE c.is_old IS TRUE AND osm_highway_linestring.osm_id = c.osm_id;
- UPDATE osm_highway_linestring_gen_z11 hl
+ UPDATE osm_highway_linestring_gen_z11
+ SET network = NULL
+ FROM transportation_name.network_changes c
+ WHERE c.is_old IS TRUE AND osm_highway_linestring_gen_z11.osm_id = c.osm_id;
+
+ UPDATE osm_highway_linestring
SET network = rm.network_type
- FROM transportation_name.network_changes c,
- osm_route_member rm
- WHERE hl.osm_id=c.osm_id
- AND hl.osm_id=rm.member
- AND rm.concurrency_index=1;
+ FROM transportation_name.network_changes c
+ JOIN transportation_route_member_coalesced rm ON (c.osm_id = rm.member AND rm.concurrency_index=1)
+ WHERE c.is_old IS FALSE AND osm_highway_linestring.osm_id=c.osm_id;
+
+ UPDATE osm_highway_linestring_gen_z11
+ SET network = rm.network_type
+ FROM transportation_name.network_changes c
+ JOIN transportation_route_member_coalesced rm ON (c.osm_id = rm.member AND rm.concurrency_index=1)
+ WHERE c.is_old IS FALSE AND osm_highway_linestring_gen_z11.osm_id=c.osm_id;
INSERT INTO osm_transportation_name_network
SELECT
@@ -318,13 +760,13 @@ BEGIN
LEAST(rm1.rank, rm2.rank, rm3.rank, rm4.rank, rm5.rank, rm6.rank) AS route_rank
FROM osm_highway_linestring hl
JOIN transportation_name.network_changes AS c ON
- hl.osm_id = c.osm_id
- LEFT OUTER JOIN osm_route_member rm1 ON rm1.member = hl.osm_id AND rm1.concurrency_index=1
- LEFT OUTER JOIN osm_route_member rm2 ON rm2.member = hl.osm_id AND rm2.concurrency_index=2
- LEFT OUTER JOIN osm_route_member rm3 ON rm3.member = hl.osm_id AND rm3.concurrency_index=3
- LEFT OUTER JOIN osm_route_member rm4 ON rm4.member = hl.osm_id AND rm4.concurrency_index=4
- LEFT OUTER JOIN osm_route_member rm5 ON rm5.member = hl.osm_id AND rm5.concurrency_index=5
- LEFT OUTER JOIN osm_route_member rm6 ON rm6.member = hl.osm_id AND rm6.concurrency_index=6
+ c.is_old IS FALSE AND hl.osm_id = c.osm_id
+ LEFT OUTER JOIN transportation_route_member_coalesced rm1 ON rm1.member = hl.osm_id AND rm1.concurrency_index=1
+ LEFT OUTER JOIN transportation_route_member_coalesced rm2 ON rm2.member = hl.osm_id AND rm2.concurrency_index=2
+ LEFT OUTER JOIN transportation_route_member_coalesced rm3 ON rm3.member = hl.osm_id AND rm3.concurrency_index=3
+ LEFT OUTER JOIN transportation_route_member_coalesced rm4 ON rm4.member = hl.osm_id AND rm4.concurrency_index=4
+ LEFT OUTER JOIN transportation_route_member_coalesced rm5 ON rm5.member = hl.osm_id AND rm5.concurrency_index=5
+ LEFT OUTER JOIN transportation_route_member_coalesced rm6 ON rm6.member = hl.osm_id AND rm6.concurrency_index=6
WHERE (hl.name <> '' OR hl.ref <> '' OR rm1.ref <> '' OR rm1.network <> '')
AND hl.highway <> ''
) AS t
@@ -366,51 +808,115 @@ CREATE CONSTRAINT TRIGGER trigger_refresh_network
FOR EACH ROW
EXECUTE PROCEDURE transportation_name.refresh_network();
--- Trigger to update "osm_transportation_name_linestring" from "osm_transportation_name_network"
+-- Handle updates on
+-- osm_transportation_name_network -> osm_transportation_name_linestring
+-- osm_shipway_linestring -> osm_transportation_name_linestring
+-- osm_aerialway_linestring -> osm_transportation_name_linestring
+-- osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1
+-- osm_transportation_name_linestring -> osm_transportation_name_linestring_gen2
+-- osm_transportation_name_linestring -> osm_transportation_name_linestring_gen3
+-- osm_transportation_name_linestring -> osm_transportation_name_linestring_gen4
+
+CREATE OR REPLACE AGGREGATE array_cat_agg(anycompatiblearray) (
+ SFUNC=array_cat,
+ STYPE=anycompatiblearray,
+ INITCOND = '{}'
+);
CREATE TABLE IF NOT EXISTS transportation_name.name_changes
(
- id serial PRIMARY KEY,
is_old boolean,
osm_id bigint,
- tags hstore,
- ref character varying,
- highway character varying,
- subclass character varying,
- brunnel character varying,
- sac_scale character varying,
- level integer,
- layer integer,
- indoor boolean,
- network_type route_network_type,
- route_1 character varying,
- route_2 character varying,
- route_3 character varying,
- route_4 character varying,
- route_5 character varying,
- route_6 character varying
+ PRIMARY KEY (is_old, osm_id)
+);
+CREATE TABLE IF NOT EXISTS transportation_name.shipway_changes
+(
+ is_old boolean,
+ osm_id bigint,
+ PRIMARY KEY (is_old, osm_id)
+);
+CREATE TABLE IF NOT EXISTS transportation_name.aerialway_changes
+(
+ is_old boolean,
+ osm_id bigint,
+ PRIMARY KEY (is_old, osm_id)
);
+-- Store IDs of changed elements from osm_transportation_name_network table.
CREATE OR REPLACE FUNCTION transportation_name.name_network_store() RETURNS trigger AS
$$
BEGIN
IF (tg_op IN ('DELETE', 'UPDATE'))
THEN
- INSERT INTO transportation_name.name_changes(is_old, osm_id, tags, ref, highway, subclass,
- brunnel, sac_scale, level, layer, indoor, network_type,
- route_1, route_2, route_3, route_4, route_5, route_6)
- VALUES (TRUE, old.osm_id, old.tags, old.ref, old.highway, old.subclass,
- old.brunnel, old.sac_scale, old.level, old.layer, old.indoor, old.network_type,
- old.route_1, old.route_2, old.route_3, old.route_4, old.route_5, old.route_6);
+ INSERT INTO transportation_name.name_changes(is_old, osm_id)
+ VALUES (TRUE, old.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
END IF;
IF (tg_op IN ('UPDATE', 'INSERT'))
THEN
- INSERT INTO transportation_name.name_changes(is_old, osm_id, tags, ref, highway, subclass,
- brunnel, sac_scale, level, layer, indoor, network_type,
- route_1, route_2, route_3, route_4, route_5, route_6)
- VALUES (FALSE, new.osm_id, new.tags, new.ref, new.highway, new.subclass,
- new.brunnel, new.sac_scale, new.level, new.layer, new.indoor, new.network_type,
- new.route_1, new.route_2, new.route_3, new.route_4, new.route_5, new.route_6);
+ INSERT INTO transportation_name.name_changes(is_old, osm_id)
+ VALUES (FALSE, new.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Store IDs of changed elements from osm_shipway_linestring table.
+CREATE OR REPLACE FUNCTION transportation_name.name_shipway_store() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op IN ('DELETE', 'UPDATE'))
+ THEN
+ INSERT INTO transportation_name.shipway_changes(is_old, osm_id)
+ VALUES (TRUE, old.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ IF (tg_op IN ('UPDATE', 'INSERT'))
+ THEN
+ INSERT INTO transportation_name.shipway_changes(is_old, osm_id)
+ VALUES (FALSE, new.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Store IDs of changed elements from osm_aerialway_linestring table.
+CREATE OR REPLACE FUNCTION transportation_name.name_aerialway_store() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op IN ('DELETE', 'UPDATE'))
+ THEN
+ INSERT INTO transportation_name.aerialway_changes(is_old, osm_id)
+ VALUES (TRUE, old.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ IF (tg_op IN ('UPDATE', 'INSERT'))
+ THEN
+ INSERT INTO transportation_name.aerialway_changes(is_old, osm_id)
+ VALUES (FALSE, new.osm_id)
+ ON CONFLICT (is_old, osm_id) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Store IDs of changed elements from osm_transportation_name_linestring table.
+CREATE OR REPLACE FUNCTION transportation_name.name_linestring_store() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op = 'DELETE')
+ THEN
+ INSERT INTO transportation_name.name_changes_gen(is_old, id)
+ VALUES (TRUE, old.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
+ END IF;
+ IF (tg_op = 'UPDATE' OR tg_op = 'INSERT')
+ THEN
+ INSERT INTO transportation_name.name_changes_gen(is_old, id)
+ VALUES (FALSE, new.id)
+ ON CONFLICT (is_old, id) DO NOTHING;
END IF;
RETURN NULL;
END;
@@ -422,6 +928,18 @@ CREATE TABLE IF NOT EXISTS transportation_name.updates_name
t text,
UNIQUE (t)
);
+CREATE TABLE IF NOT EXISTS transportation_name.updates_shipway
+(
+ id serial PRIMARY KEY,
+ t text,
+ UNIQUE (t)
+);
+CREATE TABLE IF NOT EXISTS transportation_name.updates_aerialway
+(
+ id serial PRIMARY KEY,
+ t text,
+ UNIQUE (t)
+);
CREATE OR REPLACE FUNCTION transportation_name.flag_name() RETURNS trigger AS
$$
BEGIN
@@ -430,6 +948,22 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION transportation_name.flag_shipway() RETURNS trigger AS
+$$
+BEGIN
+ INSERT INTO transportation_name.updates_shipway(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION transportation_name.flag_aerialway() RETURNS trigger AS
+$$
+BEGIN
+ INSERT INTO transportation_name.updates_aerialway(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
CREATE OR REPLACE FUNCTION transportation_name.refresh_name() RETURNS trigger AS
$BODY$
DECLARE
@@ -437,264 +971,557 @@ DECLARE
BEGIN
RAISE LOG 'Refresh transportation_name';
- -- REFRESH osm_transportation_name_linestring
+ -- REFRESH osm_transportation_name_linestring from osm_transportation_name_network
- -- Compact the change history to keep only the first and last version, and then uniq version of row
- CREATE TEMP TABLE name_changes_compact AS
- SELECT DISTINCT ON (tags, ref, highway, subclass, brunnel, sac_scale, level, layer, indoor, network_type,
- route_1, route_2, route_3, route_4, route_5, route_6)
- tags,
- ref,
- highway,
- subclass,
- brunnel,
- sac_scale,
- level,
- layer,
- indoor,
- network_type,
- route_1, route_2, route_3, route_4, route_5, route_6,
- coalesce(tags->'name', ref) AS name_ref
- FROM ((
- SELECT DISTINCT ON (osm_id) *
- FROM transportation_name.name_changes
- WHERE is_old
- ORDER BY osm_id,
- id ASC
- )
- UNION ALL
- (
- SELECT DISTINCT ON (osm_id) *
- FROM transportation_name.name_changes
- WHERE NOT is_old
- ORDER BY osm_id,
- id DESC
- )) AS t;
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation_name.name_changes;
+ ANALYZE osm_transportation_name_network;
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_transportation_name_linestring_source_ids m
+ WHERE m.source = 0 AND EXISTS(
+ SELECT NULL
+ FROM transportation_name.name_changes c
+ WHERE c.is_old IS TRUE AND c.osm_id = m.source_id
+ )
+ GROUP BY id;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
DELETE
- FROM osm_transportation_name_linestring AS n
- USING name_changes_compact AS c
- WHERE coalesce(n.ref, '') = coalesce(c.ref, '')
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.sac_scale IS NOT DISTINCT FROM c.sac_scale
- AND n.level IS NOT DISTINCT FROM c.level
- AND n.layer IS NOT DISTINCT FROM c.layer
- AND n.indoor IS NOT DISTINCT FROM c.indoor
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ FROM osm_transportation_name_linestring m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
- INSERT INTO osm_transportation_name_linestring
- SELECT (ST_Dump(geometry)).geom AS geometry,
- tags|| get_basic_names(tags, geometry) AS tags,
- ref,
- highway,
- subclass,
- brunnel,
- sac_scale,
- level,
- layer,
- indoor,
- network_type AS network,
- route_1, route_2, route_3, route_4, route_5, route_6,
- z_order
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_transportation_name_linestring;
+ ANALYZE osm_transportation_name_linestring_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT osm_id, NULL::INTEGER AS id, NULL::BIGINT[] AS source_ids, geometry, tags, ref, highway, subclass, brunnel,
+ sac_scale, level, layer, indoor, network_type, route_1, route_2, route_3, route_4, route_5, route_6,
+ z_order, route_rank
FROM (
- SELECT ST_LineMerge(ST_Collect(n.geometry)) AS geometry,
- n.tags,
- n.ref,
- n.highway,
- n.subclass,
- n.brunnel,
- n.sac_scale,
- n.level,
- n.layer,
- n.indoor,
- n.network_type,
- n.route_1, n.route_2, n.route_3, n.route_4, n.route_5, n.route_6,
- min(n.z_order) AS z_order
- FROM osm_transportation_name_network AS n
- JOIN name_changes_compact AS c ON
- coalesce(n.ref, '') = coalesce(c.ref, '')
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.sac_scale IS NOT DISTINCT FROM c.sac_scale
- AND n.level IS NOT DISTINCT FROM c.level
- AND n.layer IS NOT DISTINCT FROM c.layer
- AND n.indoor IS NOT DISTINCT FROM c.indoor
- AND n.network_type IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6
- GROUP BY n.tags, n.ref, n.highway, n.subclass, n.brunnel, n.sac_scale, n.level, n.layer, n.indoor, n.network_type,
- n.route_1, n.route_2, n.route_3, n.route_4, n.route_5, n.route_6
- ) AS highway_union;
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id
+ FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT osm_id AS source_id FROM transportation_name.name_changes WHERE is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_transportation_name_network ON (
+ affected_source_linestrings.source_id = osm_transportation_name_network.osm_id
+ )
+ WHERE coalesce(tags->'name', '') <> '' OR coalesce(ref, '') <> '';
- -- REFRESH osm_transportation_name_linestring_gen1
- DELETE FROM osm_transportation_name_linestring_gen1 AS n
- USING name_changes_compact AS c
- WHERE
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
- INSERT INTO osm_transportation_name_linestring_gen1
- SELECT n.*
- FROM osm_transportation_name_linestring_gen1_view AS n
- JOIN name_changes_compact AS c ON
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
- -- REFRESH osm_transportation_name_linestring_gen2
- DELETE FROM osm_transportation_name_linestring_gen2 AS n
- USING name_changes_compact AS c
- WHERE
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::BIGINT AS osm_id, m.id,
+ ARRAY(
+ SELECT s.source_id
+ FROM osm_transportation_name_linestring_source_ids s
+ WHERE s.source = 0 AND m.id = s.id
+ )::BIGINT[] AS source_ids,
+ m.geometry, m.tags, m.ref, m.highway, m.subclass, m.brunnel, m.sac_scale,
+ m.level, m.layer, m.indoor, m.network AS network_type, m.route_1, m.route_2, m.route_3,
+ m.route_4, m.route_5, m.route_6, m.z_order, m.route_rank
+ FROM linestrings_to_merge
+ JOIN osm_transportation_name_linestring m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry))
+ WHERE m.source = 0;
- INSERT INTO osm_transportation_name_linestring_gen2
- SELECT n.*
- FROM osm_transportation_name_linestring_gen2_view AS n
- JOIN name_changes_compact AS c ON
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
- -- REFRESH osm_transportation_name_linestring_gen3
- DELETE FROM osm_transportation_name_linestring_gen3 AS n
- USING name_changes_compact AS c
- WHERE
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_transportation_name_linestring m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
- INSERT INTO osm_transportation_name_linestring_gen3
- SELECT n.*
- FROM osm_transportation_name_linestring_gen3_view AS n
- JOIN name_changes_compact AS c ON
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1.
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor, network_type,
+ route_1, route_2, route_3, route_4, route_5, route_6
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (
+ ORDER BY tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor, network_type, route_1,
+ route_2, route_3, route_4, route_5, route_6
+ ) as cluster_group
+ FROM linestrings_to_merge;
- -- REFRESH osm_transportation_name_linestring_gen4
- DELETE FROM osm_transportation_name_linestring_gen4 AS n
- USING name_changes_compact AS c
- WHERE
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
- INSERT INTO osm_transportation_name_linestring_gen4
- SELECT n.*
- FROM osm_transportation_name_linestring_gen4_view AS n
- JOIN name_changes_compact AS c ON
- coalesce(n.tags->'name', n.ref) = c.name_ref
- AND coalesce(n.tags, '') = coalesce(c.tags, '')
- AND n.ref IS NOT DISTINCT FROM c.ref
- AND n.highway IS NOT DISTINCT FROM c.highway
- AND n.subclass IS NOT DISTINCT FROM c.subclass
- AND n.brunnel IS NOT DISTINCT FROM c.brunnel
- AND n.network IS NOT DISTINCT FROM c.network_type
- AND n.route_1 IS NOT DISTINCT FROM c.route_1
- AND n.route_2 IS NOT DISTINCT FROM c.route_2
- AND n.route_3 IS NOT DISTINCT FROM c.route_3
- AND n.route_4 IS NOT DISTINCT FROM c.route_4
- AND n.route_5 IS NOT DISTINCT FROM c.route_5
- AND n.route_6 IS NOT DISTINCT FROM c.route_6;
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
- DROP TABLE name_changes_compact;
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS new_source_ids BIGINT[];
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS old_source_ids BIGINT[];
+
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_transportation_name_linestring(source, geometry, new_source_ids, old_source_ids, tags, ref,
+ highway, subclass, brunnel, sac_scale, "level", layer, indoor,
+ network, route_1, route_2, route_3, route_4, route_5, route_6,
+ z_order, route_rank)
+ SELECT 0 AS source, (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries
+ -- to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString
+ -- if more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(osm_id) FILTER (WHERE osm_id IS NOT NULL), '{}' )::BIGINT[] AS new_source_ids,
+ array_cat_agg(source_ids)::BIGINT[] as old_source_ids,
+ tags, ref, highway, subclass, brunnel, sac_scale, level, layer,
+ indoor, network_type, route_1, route_2, route_3, route_4, route_5, route_6,
+ min(z_order) AS z_order, min(route_rank) AS route_rank
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, tags, ref, highway, subclass, brunnel, level, layer, sac_scale, indoor,
+ network_type, route_1, route_2, route_3, route_4, route_5, route_6
+ RETURNING source, id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources.
+ -- This is required because ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_transportation_name_linestring_source_ids (source, id, source_id)
+ SELECT m.source, m.id, source_id
+ FROM (
+ SELECT source, id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_transportation_name_network s ON (m.source_id = s.osm_id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (source, id, source_id) DO NOTHING;
+
+ -- Cleanup remaining table
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS old_source_ids;
+
+ -- noinspection SqlWithoutWhere
DELETE FROM transportation_name.name_changes;
+ -- noinspection SqlWithoutWhere
DELETE FROM transportation_name.updates_name;
RAISE LOG 'Refresh transportation_name done in %', age(clock_timestamp(), t);
+
+ -- Update gen1, gen2, gen3 and gen4 tables
+ PERFORM update_transportation_name_linestring_gen(FALSE);
+
RETURN NULL;
END;
-$BODY$
- LANGUAGE plpgsql;
+$BODY$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION transportation_name.refresh_shipway_linestring() RETURNS trigger AS
+$BODY$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh transportation_name shiwpway';
+
+ -- REFRESH osm_transportation_name_linestring from osm_shipway_linestring
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation_name.shipway_changes;
+ ANALYZE osm_shipway_linestring;
+
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_transportation_name_linestring_source_ids m
+ WHERE m.source = 1 AND EXISTS(
+ SELECT NULL
+ FROM transportation_name.shipway_changes c
+ WHERE c.is_old IS TRUE AND c.osm_id = m.source_id
+ )
+ GROUP BY id;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
+ DELETE
+ FROM osm_transportation_name_linestring m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_transportation_name_linestring;
+ ANALYZE osm_transportation_name_linestring_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT osm_id, NULL::INTEGER AS id, NULL::BIGINT[] AS source_ids, geometry,
+ transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ) AS tags, shipway AS subclass, layer, z_order
+ FROM (
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id
+ FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT osm_id AS source_id FROM transportation_name.shipway_changes WHERE is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_shipway_linestring ON (
+ affected_source_linestrings.source_id = osm_shipway_linestring.osm_id
+ )
+ WHERE name <> '';
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::BIGINT AS osm_id, m.id,
+ ARRAY(
+ SELECT s.source_id
+ FROM osm_transportation_name_linestring_source_ids s
+ WHERE s.source = 1 AND m.id = s.id
+ )::BIGINT[] AS source_ids,
+ m.geometry, m.tags, m.subclass, m.layer, m.z_order
+ FROM linestrings_to_merge
+ JOIN osm_transportation_name_linestring m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry))
+ WHERE m.source = 1;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_transportation_name_linestring m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1.
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (PARTITION BY tags, subclass, layer) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (ORDER BY tags, subclass, layer) as cluster_group
+ FROM linestrings_to_merge;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
+
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
+
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS new_source_ids BIGINT[];
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS old_source_ids BIGINT[];
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_transportation_name_linestring(source, geometry, new_source_ids, old_source_ids, tags, highway,
+ subclass, z_order)
+ SELECT 1 AS source, (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries
+ -- to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString
+ -- if more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(osm_id) FILTER (WHERE osm_id IS NOT NULL), '{}' )::BIGINT[] AS new_source_ids,
+ array_cat_agg(source_ids)::BIGINT[] as old_source_ids,
+ tags, 'shipway' AS highway, subclass, min(z_order) AS z_order
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, tags, subclass, layer
+ RETURNING source, id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources.
+ -- This is required because ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_transportation_name_linestring_source_ids (source, id, source_id)
+ SELECT m.source, m.id, source_id
+ FROM (
+ SELECT source, id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_shipway_linestring s ON (m.source_id = s.osm_id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (source, id, source_id) DO NOTHING;
+
+ -- Cleanup remaining table
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS old_source_ids;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation_name.shipway_changes;
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation_name.updates_shipway;
+
+ RAISE LOG 'Refresh transportation_name shipway done in %', age(clock_timestamp(), t);
+
+ -- Update gen1, gen2, gen3 and gen4 tables
+ PERFORM update_transportation_name_linestring_gen(FALSE);
+
+ RETURN NULL;
+END;
+$BODY$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION transportation_name.refresh_aerialway_linestring() RETURNS trigger AS
+$BODY$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh transportation_name aerialway';
+
+ -- REFRESH osm_transportation_name_linestring from osm_aerialway_linestring
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE transportation_name.aerialway_changes;
+ ANALYZE osm_aerialway_linestring;
+
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_transportation_name_linestring_source_ids m
+ WHERE m.source = 2 AND EXISTS(
+ SELECT NULL
+ FROM transportation_name.aerialway_changes c
+ WHERE c.is_old IS TRUE AND c.osm_id = m.source_id
+ )
+ GROUP BY id;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
+ DELETE
+ FROM osm_transportation_name_linestring m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_transportation_name_linestring;
+ ANALYZE osm_transportation_name_linestring_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT osm_id, NULL::INTEGER AS id, NULL::BIGINT[] AS source_ids, geometry,
+ transportation_name_tags(
+ NULL::geometry, tags, name, name_en, name_de
+ ) AS tags, aerialway AS subclass, layer, z_order
+ FROM (
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id
+ FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT osm_id AS source_id FROM transportation_name.aerialway_changes WHERE is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_aerialway_linestring ON (
+ affected_source_linestrings.source_id = osm_aerialway_linestring.osm_id
+ )
+ WHERE name <> '';
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::BIGINT AS osm_id, m.id,
+ ARRAY(
+ SELECT s.source_id
+ FROM osm_transportation_name_linestring_source_ids s
+ WHERE s.source = 2 AND m.id = s.id
+ )::BIGINT[] AS source_ids,
+ m.geometry, m.tags, m.subclass, m.layer, m.z_order
+ FROM linestrings_to_merge
+ JOIN osm_transportation_name_linestring m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry))
+ WHERE m.source = 2;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_transportation_name_linestring m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_transportation_name_linestring_source_ids m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1.
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (PARTITION BY tags, subclass, layer) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (ORDER BY tags, subclass, layer) as cluster_group
+ FROM linestrings_to_merge;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
+
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
+
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS new_source_ids BIGINT[];
+ ALTER TABLE osm_transportation_name_linestring ADD COLUMN IF NOT EXISTS old_source_ids BIGINT[];
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_transportation_name_linestring(source, geometry, new_source_ids, old_source_ids, tags, highway,
+ subclass, z_order)
+ SELECT 2 AS source, (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries
+ -- to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString
+ -- if more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(osm_id) FILTER (WHERE osm_id IS NOT NULL), '{}' )::BIGINT[] AS new_source_ids,
+ array_cat_agg(source_ids)::BIGINT[] as old_source_ids,
+ tags, 'aerialway' AS highway, subclass, min(z_order) AS z_order
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, tags, subclass, layer
+ RETURNING source, id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources.
+ -- This is required because ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_transportation_name_linestring_source_ids (source, id, source_id)
+ SELECT m.source, m.id, source_id
+ FROM (
+ SELECT source, id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_aerialway_linestring s ON (m.source_id = s.osm_id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (source, id, source_id) DO NOTHING;
+
+ -- Cleanup remaining table
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_transportation_name_linestring DROP COLUMN IF EXISTS old_source_ids;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation_name.aerialway_changes;
+ -- noinspection SqlWithoutWhere
+ DELETE FROM transportation_name.updates_aerialway;
+
+ RAISE LOG 'Refresh transportation_name aerialway done in %', age(clock_timestamp(), t);
+
+ -- Update gen1, gen2, gen3 and gen4 tables
+ PERFORM update_transportation_name_linestring_gen(FALSE);
+
+ RETURN NULL;
+END;
+$BODY$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store_transportation_name_network
AFTER INSERT OR UPDATE OR DELETE
@@ -702,15 +1529,59 @@ CREATE TRIGGER trigger_store_transportation_name_network
FOR EACH ROW
EXECUTE PROCEDURE transportation_name.name_network_store();
+CREATE TRIGGER trigger_store_transportation_name_shipway
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_shipway_linestring
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation_name.name_shipway_store();
+
+CREATE TRIGGER trigger_store_transportation_name_aerialway
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_aerialway_linestring
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation_name.name_aerialway_store();
+
+CREATE TRIGGER trigger_store_transportation_name_linestring
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_transportation_name_linestring
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation_name.name_linestring_store();
+
CREATE TRIGGER trigger_flag_name
AFTER INSERT
ON transportation_name.name_changes
FOR EACH STATEMENT
EXECUTE PROCEDURE transportation_name.flag_name();
+CREATE TRIGGER trigger_flag_shipway
+ AFTER INSERT
+ ON transportation_name.shipway_changes
+ FOR EACH STATEMENT
+EXECUTE PROCEDURE transportation_name.flag_shipway();
+
+CREATE TRIGGER trigger_flag_aerialway
+ AFTER INSERT
+ ON transportation_name.aerialway_changes
+ FOR EACH STATEMENT
+EXECUTE PROCEDURE transportation_name.flag_aerialway();
+
CREATE CONSTRAINT TRIGGER trigger_refresh_name
AFTER INSERT
ON transportation_name.updates_name
INITIALLY DEFERRED
FOR EACH ROW
EXECUTE PROCEDURE transportation_name.refresh_name();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh_shipway
+ AFTER INSERT
+ ON transportation_name.updates_shipway
+ INITIALLY DEFERRED
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation_name.refresh_shipway_linestring();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh_aerialway
+ AFTER INSERT
+ ON transportation_name.updates_aerialway
+ INITIALLY DEFERRED
+ FOR EACH ROW
+EXECUTE PROCEDURE transportation_name.refresh_aerialway_linestring();
diff --git a/layers/water_name/update_marine_point.sql b/layers/water_name/update_marine_point.sql
index 5fdbc7a2..95dfd962 100644
--- a/layers/water_name/update_marine_point.sql
+++ b/layers/water_name/update_marine_point.sql
@@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS water_name_marine;
CREATE TABLE IF NOT EXISTS water_name_marine.osm_ids
(
- osm_id bigint
+ osm_id bigint PRIMARY KEY
);
CREATE OR REPLACE FUNCTION update_osm_marine_point(full_update boolean) RETURNS void AS
@@ -45,11 +45,7 @@ CREATE INDEX IF NOT EXISTS osm_marine_point_rank_idx ON osm_marine_point ("rank"
CREATE OR REPLACE FUNCTION water_name_marine.store() RETURNS trigger AS
$$
BEGIN
- IF (tg_op = 'DELETE') THEN
- INSERT INTO water_name_marine.osm_ids VALUES (OLD.osm_id);
- ELSE
- INSERT INTO water_name_marine.osm_ids VALUES (NEW.osm_id);
- END IF;
+ INSERT INTO water_name_marine.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
@@ -74,6 +70,11 @@ DECLARE
t TIMESTAMP WITH TIME ZONE := clock_timestamp();
BEGIN
RAISE LOG 'Refresh water_name_marine rank';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE water_name_marine.osm_ids;
+ ANALYZE osm_marine_point;
+
PERFORM update_osm_marine_point(false);
-- noinspection SqlWithoutWhere
DELETE FROM water_name_marine.osm_ids;
@@ -86,13 +87,13 @@ END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trigger_store
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_marine_point
FOR EACH ROW
EXECUTE PROCEDURE water_name_marine.store();
CREATE TRIGGER trigger_flag
- AFTER INSERT OR UPDATE OR DELETE
+ AFTER INSERT OR UPDATE
ON osm_marine_point
FOR EACH STATEMENT
EXECUTE PROCEDURE water_name_marine.flag();
diff --git a/layers/water_name/update_water_lakeline.sql b/layers/water_name/update_water_lakeline.sql
deleted file mode 100644
index fccc317e..00000000
--- a/layers/water_name/update_water_lakeline.sql
+++ /dev/null
@@ -1,94 +0,0 @@
-DROP TRIGGER IF EXISTS trigger_delete_line ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_update_line ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_insert_line ON osm_water_polygon;
-
-CREATE OR REPLACE VIEW osm_water_lakeline_view AS
-SELECT wp.osm_id,
- ll.wkb_geometry AS geometry,
- name,
- name_en,
- name_de,
- update_tags(tags, ll.wkb_geometry) AS tags,
- ST_Area(wp.geometry) AS area,
- is_intermittent
-FROM osm_water_polygon AS wp
- INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
-WHERE wp.name <> ''
- AND ST_IsValid(wp.geometry);
-
--- etldoc: osm_water_polygon -> osm_water_lakeline
--- etldoc: lake_centerline -> osm_water_lakeline
-CREATE TABLE IF NOT EXISTS osm_water_lakeline AS
-SELECT *
-FROM osm_water_lakeline_view;
-DO
-$$
- BEGIN
- ALTER TABLE osm_water_lakeline
- ADD CONSTRAINT osm_water_lakeline_pk PRIMARY KEY (osm_id);
- EXCEPTION
- WHEN OTHERS THEN
- RAISE NOTICE 'primary key osm_water_lakeline_pk already exists in osm_water_lakeline.';
- END;
-$$;
-CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist (geometry);
-
--- Handle updates
-
-CREATE SCHEMA IF NOT EXISTS water_lakeline;
-
-CREATE OR REPLACE FUNCTION water_lakeline.delete() RETURNS trigger AS
-$$
-BEGIN
- DELETE
- FROM osm_water_lakeline
- WHERE osm_water_lakeline.osm_id = OLD.osm_id;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION water_lakeline.update() RETURNS trigger AS
-$$
-BEGIN
- UPDATE osm_water_lakeline
- SET (osm_id, geometry, name, name_en, name_de, tags, area, is_intermittent) =
- (SELECT * FROM osm_water_lakeline_view WHERE osm_water_lakeline_view.osm_id = NEW.osm_id)
- WHERE osm_water_lakeline.osm_id = NEW.osm_id;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION water_lakeline.insert() RETURNS trigger AS
-$$
-BEGIN
- INSERT INTO osm_water_lakeline
- SELECT *
- FROM osm_water_lakeline_view
- WHERE osm_water_lakeline_view.osm_id = NEW.osm_id
- -- May happen in case we replay update
- ON CONFLICT ON CONSTRAINT osm_water_lakeline_pk
- DO NOTHING;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER trigger_delete_line
- AFTER DELETE
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_lakeline.delete();
-
-CREATE TRIGGER trigger_update_line
- AFTER UPDATE
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_lakeline.update();
-
-CREATE TRIGGER trigger_insert_line
- AFTER INSERT
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_lakeline.insert();
diff --git a/layers/water_name/update_water_name.sql b/layers/water_name/update_water_name.sql
new file mode 100644
index 00000000..b37ed761
--- /dev/null
+++ b/layers/water_name/update_water_name.sql
@@ -0,0 +1,213 @@
+DROP TRIGGER IF EXISTS trigger_store ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon;
+DROP TRIGGER IF EXISTS trigger_refresh ON water_name.updates;
+
+CREATE INDEX IF NOT EXISTS lake_centerline_osm_id_idx ON lake_centerline (osm_id);
+CREATE INDEX IF NOT EXISTS osm_water_polygon_update_idx ON osm_water_polygon (name, ST_IsValid(geometry))
+ WHERE name <> '' AND ST_IsValid(geometry);;
+
+CREATE OR REPLACE VIEW osm_water_lakeline_view AS
+SELECT wp.osm_id,
+ ll.wkb_geometry AS geometry,
+ name,
+ name_en,
+ name_de,
+ update_tags(tags, ll.wkb_geometry) AS tags,
+ ST_Area(wp.geometry) AS area,
+ is_intermittent
+FROM osm_water_polygon AS wp
+ INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
+WHERE wp.name <> ''
+ AND ST_IsValid(wp.geometry);
+
+-- etldoc: osm_water_polygon -> osm_water_lakeline
+-- etldoc: lake_centerline -> osm_water_lakeline
+CREATE TABLE IF NOT EXISTS osm_water_lakeline AS
+SELECT *
+FROM osm_water_lakeline_view;
+DO
+$$
+ BEGIN
+ ALTER TABLE osm_water_lakeline
+ ADD CONSTRAINT osm_water_lakeline_pk PRIMARY KEY (osm_id);
+ EXCEPTION
+ WHEN OTHERS THEN
+ RAISE NOTICE 'primary key osm_water_lakeline_pk already exists in osm_water_lakeline.';
+ END;
+$$;
+CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist (geometry);
+
+-- etldoc: osm_water_polygon -> osm_water_point_view
+-- etldoc: lake_centerline -> osm_water_point_view
+CREATE OR REPLACE VIEW osm_water_point_view AS
+SELECT wp.osm_id,
+ ST_PointOnSurface(wp.geometry) AS geometry,
+ wp.name,
+ wp.name_en,
+ wp.name_de,
+ CASE
+ WHEN "natural" = 'bay' THEN 'bay'
+ WHEN place = 'sea' THEN 'sea'
+ ELSE 'lake'
+ END AS class,
+ update_tags(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags,
+ -- Area of the feature in square meters
+ ST_Area(wp.geometry) AS area,
+ wp.is_intermittent
+FROM osm_water_polygon AS wp
+ LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
+WHERE ll.osm_id IS NULL
+ AND wp.name <> ''
+ AND ST_IsValid(wp.geometry);
+
+-- etldoc: osm_water_point_view -> osm_water_point_earth_view
+CREATE OR REPLACE VIEW osm_water_point_earth_view AS
+SELECT osm_id,
+ geometry,
+ name,
+ name_en,
+ name_de,
+ class,
+ tags,
+ -- Percentage of the earth's surface covered by this feature (approximately)
+ -- The constant below is 111,842^2 * 180 * 180, where 111,842 is the length of one degree of latitude at the equator in meters.
+ area / (405279708033600 * COS(ST_Y(ST_Transform(geometry,4326))*PI()/180)) as earth_area,
+ is_intermittent
+FROM osm_water_point_view;
+
+-- etldoc: osm_water_point_earth_view -> osm_water_point
+CREATE TABLE IF NOT EXISTS osm_water_point AS
+SELECT *
+FROM osm_water_point_earth_view;
+DO
+$$
+ BEGIN
+ ALTER TABLE osm_water_point
+ ADD CONSTRAINT osm_water_point_pk PRIMARY KEY (osm_id);
+ EXCEPTION
+ WHEN OTHERS THEN
+ RAISE NOTICE 'primary key osm_water_point_pk already exists in osm_water_point.';
+ END;
+$$;
+CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry);
+
+-- Handle updates
+
+CREATE SCHEMA IF NOT EXISTS water_name;
+
+CREATE TABLE IF NOT EXISTS water_name.osm_ids
+(
+ osm_id bigint,
+ is_old bool,
+ PRIMARY KEY (osm_id, is_old)
+);
+
+CREATE OR REPLACE FUNCTION update_osm_water_name() RETURNS void AS $$
+BEGIN
+ DELETE FROM osm_water_lakeline
+ WHERE EXISTS(
+ SELECT NULL
+ FROM water_name.osm_ids
+ WHERE water_name.osm_ids.osm_id = osm_water_lakeline.osm_id
+ AND water_name.osm_ids.is_old IS TRUE
+ );
+
+ INSERT INTO osm_water_lakeline
+ SELECT * FROM osm_water_lakeline_view
+ WHERE EXISTS(
+ SELECT NULL
+ FROM water_name.osm_ids
+ WHERE water_name.osm_ids.osm_id = osm_water_lakeline_view.osm_id
+ AND water_name.osm_ids.is_old IS FALSE
+ ) ON CONFLICT (osm_id) DO UPDATE SET geometry = excluded.geometry, name = excluded.name, name_en = excluded.name_en,
+ name_de = excluded.name_de, tags = excluded.tags, area = excluded.area,
+ is_intermittent = excluded.is_intermittent;
+
+ DELETE FROM osm_water_point
+ WHERE EXISTS(
+ SELECT NULL
+ FROM water_name.osm_ids
+ WHERE water_name.osm_ids.osm_id = osm_water_point.osm_id
+ AND water_name.osm_ids.is_old IS TRUE
+ );
+
+ INSERT INTO osm_water_point
+ SELECT * FROM osm_water_point_earth_view
+ WHERE EXISTS(
+ SELECT NULL
+ FROM water_name.osm_ids
+ WHERE water_name.osm_ids.osm_id = osm_water_point_earth_view.osm_id
+ AND water_name.osm_ids.is_old IS FALSE
+ ) ON CONFLICT (osm_id) DO UPDATE SET geometry = excluded.geometry, name = excluded.name, name_en = excluded.name_en,
+ name_de = excluded.name_de, class = excluded.class, tags = excluded.tags,
+ earth_area = excluded.earth_area, is_intermittent = excluded.is_intermittent;
+
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION water_name.store() RETURNS trigger AS $$
+BEGIN
+ IF (tg_op = 'DELETE') THEN
+ INSERT INTO water_name.osm_ids (osm_id, is_old) VALUES (OLD.osm_id, TRUE) ON CONFLICT (osm_id, is_old) DO NOTHING;
+ ELSE
+ INSERT INTO water_name.osm_ids (osm_id, is_old) VALUES (NEW.osm_id, FALSE) ON CONFLICT (osm_id, is_old) DO NOTHING;
+ END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TABLE IF NOT EXISTS water_name.updates
+(
+ id serial PRIMARY KEY,
+ t text,
+ UNIQUE (t)
+);
+CREATE OR REPLACE FUNCTION water_name.flag() RETURNS trigger AS
+$$
+BEGIN
+ INSERT INTO water_name.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS
+$$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh water_name';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE water_name.osm_ids;
+ ANALYZE osm_water_lakeline;
+ ANALYZE osm_water_point;
+
+ PERFORM update_osm_water_name();
+ -- noinspection SqlWithoutWhere
+ DELETE FROM water_name.osm_ids;
+ -- noinspection SqlWithoutWhere
+ DELETE FROM water_name.updates;
+
+ RAISE LOG 'Refresh water_name done in %', age(clock_timestamp(), t);
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_store
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_water_polygon
+ FOR EACH ROW
+EXECUTE PROCEDURE water_name.store();
+
+CREATE TRIGGER trigger_flag
+ AFTER INSERT OR UPDATE OR DELETE
+ ON osm_water_polygon
+ FOR EACH STATEMENT
+EXECUTE PROCEDURE water_name.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+ AFTER INSERT
+ ON water_name.updates
+ INITIALLY DEFERRED
+ FOR EACH ROW
+EXECUTE PROCEDURE water_name.refresh();
diff --git a/layers/water_name/update_water_point.sql b/layers/water_name/update_water_point.sql
deleted file mode 100644
index 2199b3fd..00000000
--- a/layers/water_name/update_water_point.sql
+++ /dev/null
@@ -1,117 +0,0 @@
-DROP TRIGGER IF EXISTS trigger_delete_point ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_update_point ON osm_water_polygon;
-DROP TRIGGER IF EXISTS trigger_insert_point ON osm_water_polygon;
-
--- etldoc: osm_water_polygon -> osm_water_point_view
--- etldoc: lake_centerline -> osm_water_point_view
-CREATE OR REPLACE VIEW osm_water_point_view AS
-SELECT wp.osm_id,
- ST_PointOnSurface(wp.geometry) AS geometry,
- wp.name,
- wp.name_en,
- wp.name_de,
- CASE
- WHEN "natural" = 'bay' THEN 'bay'
- WHEN place = 'sea' THEN 'sea'
- ELSE 'lake'
- END AS class,
- update_tags(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags,
- -- Area of the feature in square meters
- ST_Area(wp.geometry) as area,
- wp.is_intermittent
-FROM osm_water_polygon AS wp
- LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id
-WHERE ll.osm_id IS NULL
- AND wp.name <> ''
- AND ST_IsValid(wp.geometry);
-
--- etldoc: osm_water_point_view -> osm_water_point_earth_view
-CREATE OR REPLACE VIEW osm_water_point_earth_view AS
-SELECT osm_id,
- geometry,
- name,
- name_en,
- name_de,
- class,
- tags,
- -- Percentage of the earth's surface covered by this feature (approximately)
- -- The constant below is 111,842^2 * 180 * 180, where 111,842 is the length of one degree of latitude at the equator in meters.
- area / (405279708033600 * COS(ST_Y(ST_Transform(geometry,4326))*PI()/180)) as earth_area,
- is_intermittent
-FROM osm_water_point_view;
-
--- etldoc: osm_water_point_earth_view -> osm_water_point
-CREATE TABLE IF NOT EXISTS osm_water_point AS
-SELECT *
-FROM osm_water_point_earth_view;
-DO
-$$
- BEGIN
- ALTER TABLE osm_water_point
- ADD CONSTRAINT osm_water_point_pk PRIMARY KEY (osm_id);
- EXCEPTION
- WHEN OTHERS THEN
- RAISE NOTICE 'primary key osm_water_point_pk already exists in osm_water_point.';
- END;
-$$;
-CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry);
-
--- Handle updates
-
-CREATE SCHEMA IF NOT EXISTS water_point;
-
-CREATE OR REPLACE FUNCTION water_point.delete() RETURNS trigger AS
-$$
-BEGIN
- DELETE
- FROM osm_water_point
- WHERE osm_water_point.osm_id = OLD.osm_id;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION water_point.update() RETURNS trigger AS
-$$
-BEGIN
- UPDATE osm_water_point
- SET (osm_id, geometry, name, name_en, name_de, tags, area, is_intermittent) =
- (SELECT * FROM osm_water_point_view WHERE osm_water_point_view.osm_id = NEW.osm_id)
- WHERE osm_water_point.osm_id = NEW.osm_id;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION water_point.insert() RETURNS trigger AS
-$$
-BEGIN
- INSERT INTO osm_water_point
- SELECT *
- FROM osm_water_point_view
- WHERE osm_water_point_view.osm_id = NEW.osm_id
- -- May happen in case we replay update
- ON CONFLICT ON CONSTRAINT osm_water_point_pk
- DO NOTHING;
-
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER trigger_delete_point
- AFTER DELETE
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_point.delete();
-
-CREATE TRIGGER trigger_update_point
- AFTER UPDATE
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_point.update();
-
-CREATE TRIGGER trigger_insert_point
- AFTER INSERT
- ON osm_water_polygon
- FOR EACH ROW
-EXECUTE PROCEDURE water_point.insert();
diff --git a/layers/water_name/water_name.yaml b/layers/water_name/water_name.yaml
index 148e9088..89b6098e 100644
--- a/layers/water_name/water_name.yaml
+++ b/layers/water_name/water_name.yaml
@@ -35,8 +35,7 @@ layer:
query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, intermittent FROM layer_water_name(!bbox!, z(!scale_denominator!))) AS t
schema:
- ./update_marine_point.sql
- - ./update_water_lakeline.sql
- - ./update_water_point.sql
+ - ./update_water_name.sql
- ./water_name.sql
datasources:
- type: imposm3
diff --git a/layers/waterway/update_important_waterway.sql b/layers/waterway/update_important_waterway.sql
index 28eabe60..134fdaf7 100644
--- a/layers/waterway/update_important_waterway.sql
+++ b/layers/waterway/update_important_waterway.sql
@@ -1,4 +1,4 @@
-DROP TRIGGER IF EXISTS trigger_important_waterway_linestring ON osm_important_waterway_linestring;
+DROP TRIGGER IF EXISTS trigger_important_waterway_linestring_store ON osm_important_waterway_linestring;
DROP TRIGGER IF EXISTS trigger_store ON osm_waterway_linestring;
DROP TRIGGER IF EXISTS trigger_flag ON osm_waterway_linestring;
DROP TRIGGER IF EXISTS trigger_refresh ON waterway_important.updates;
@@ -7,54 +7,183 @@ DROP TRIGGER IF EXISTS trigger_refresh ON waterway_important.updates;
-- This helps to drop not important rivers (since they do not have a name)
-- and also makes it possible to filter out too short rivers
-CREATE INDEX IF NOT EXISTS osm_waterway_linestring_waterway_partial_idx
- ON osm_waterway_linestring ((true))
+-- Index for filling and updating osm_important_waterway_linestring table
+CREATE UNIQUE INDEX IF NOT EXISTS osm_waterway_linestring_waterway_partial_idx
+ ON osm_waterway_linestring (osm_id)
WHERE name <> ''
AND waterway = 'river'
AND ST_IsValid(geometry);
+-- Analyze created index
+ANALYZE osm_waterway_linestring;
+
CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring (
- id SERIAL PRIMARY KEY,
- geometry geometry,
+ id SERIAL,
+ geometry geometry('LineString'),
+ source_ids bigint[],
name varchar,
name_en varchar,
name_de varchar,
tags hstore
);
--- etldoc: osm_waterway_linestring -> osm_important_waterway_linestring
-INSERT INTO osm_important_waterway_linestring (geometry, name, name_en, name_de, tags)
-SELECT (ST_Dump(geometry)).geom AS geometry,
- name,
- name_en,
- name_de,
- tags
-FROM (
- SELECT ST_LineMerge(ST_Union(geometry)) AS geometry,
- name,
- name_en,
- name_de,
- slice_language_tags(tags) AS tags
- FROM osm_waterway_linestring
- WHERE name <> ''
- AND waterway = 'river'
- AND ST_IsValid(geometry)
- GROUP BY name, name_en, name_de, slice_language_tags(tags)
- ) AS waterway_union;
-CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx ON osm_important_waterway_linestring USING gist (geometry);
-
+-- Create osm_important_waterway_linestring_gen_z11 as a copy of osm_important_waterway_linestring but drop the
+-- "source_ids" column. This can be done because z10 and z9 tables are only simplified and not merged, therefore
+-- relations to sources are direct via the id column.
CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen_z11
(LIKE osm_important_waterway_linestring);
+ALTER TABLE osm_important_waterway_linestring_gen_z11 DROP COLUMN IF EXISTS source_ids;
+-- Create osm_important_waterway_linestring_gen_z10 as a copy of osm_important_waterway_linestring_gen_z11
CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen_z10
(LIKE osm_important_waterway_linestring_gen_z11);
+-- Create osm_important_waterway_linestring_gen_z9 as a copy of osm_important_waterway_linestring_gen_z10
CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen_z9
(LIKE osm_important_waterway_linestring_gen_z10);
-CREATE OR REPLACE FUNCTION insert_important_waterway_linestring_gen(update_id bigint) RETURNS void AS
-$$
+-- Create OneToMany-Relation-Table storing relations of a Merged-LineString in table
+-- osm_important_waterway_linestring to Source-LineStrings from table osm_waterway_linestring
+CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_source_ids(
+ id int,
+ source_id bigint,
+ PRIMARY KEY (id, source_id)
+);
+
+-- Ensure tables are emtpy if they haven't been created
+TRUNCATE osm_important_waterway_linestring;
+TRUNCATE osm_important_waterway_linestring_source_ids;
+
+-- etldoc: osm_waterway_linestring -> osm_important_waterway_linestring
+-- Merge LineStrings from osm_waterway_linestring by grouping them and creating intersecting
+-- clusters of each group via ST_ClusterDBSCAN
+INSERT INTO osm_important_waterway_linestring (geometry, source_ids, name, name_en, name_de, tags)
+SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries
+ -- to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString
+ -- if more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ array_agg(osm_id) as source_ids,
+ name,
+ name_en,
+ name_de,
+ slice_language_tags(tags) AS tags
+FROM (
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1.
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (
+ PARTITION BY name, name_en, name_de, slice_language_tags(tags)
+ ) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (ORDER BY name, name_en, name_de, slice_language_tags(tags)) as cluster_group
+ FROM osm_waterway_linestring
+ WHERE name <> '' AND waterway = 'river' AND ST_IsValid(geometry)
+) q
+GROUP BY cluster_group, cluster, name, name_en, name_de, slice_language_tags(tags);
+
+-- Geometry Index
+CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx
+ ON osm_important_waterway_linestring USING gist (geometry);
+
+-- Create Primary-Keys for osm_important_waterway_linestring and osm_important_waterway_linestring_gen_z11/z10/z9 tables
+DO $$
BEGIN
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_important_waterway_linestring' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_important_waterway_linestring ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_important_waterway_linestring_gen_z11' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_important_waterway_linestring_gen_z11 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_important_waterway_linestring_gen_z10' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_important_waterway_linestring_gen_z10 ADD PRIMARY KEY (id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT constraint_name
+ FROM information_schema.table_constraints
+ WHERE table_name = 'osm_important_waterway_linestring_gen_z9' AND constraint_type = 'PRIMARY KEY'
+ ) THEN
+ ALTER TABLE osm_important_waterway_linestring_gen_z9 ADD PRIMARY KEY (id);
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Index for storing OSM-IDs of Source-LineStrings
+CREATE UNIQUE INDEX IF NOT EXISTS osm_waterway_linestring_osm_id_idx ON osm_waterway_linestring ("osm_id");
+
+-- Indexes which can be utilized during full-update for queries originating from
+-- insert_important_waterway_linestring_gen() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_important_waterway_linestring_update_idx
+ ON osm_important_waterway_linestring (id) WHERE ST_Length(geometry) > 1000;
+
+-- Analyze populated table with indexes
+ANALYZE osm_important_waterway_linestring;
+
+-- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources. This required because
+-- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString if
+-- more than two LineStrings form an intersection or no intersection could be found.
+-- Execute after indexes have been created on osm_highway_linestring_gen_z11 to improve performance
+INSERT INTO osm_important_waterway_linestring_source_ids (id, source_id)
+SELECT m.id, m.source_id
+FROM (
+ SELECT id, unnest(source_ids) AS source_id, geometry
+ FROM osm_important_waterway_linestring
+) m
+JOIN osm_waterway_linestring s ON (m.source_id = s.osm_id)
+WHERE ST_Intersects(s.geometry, m.geometry)
+ON CONFLICT (id, source_id) DO NOTHING;
+
+-- Drop temporary Merged-LineString to Source-LineStrings-ID column
+ALTER TABLE osm_important_waterway_linestring DROP COLUMN IF EXISTS source_ids;
+
+CREATE SCHEMA IF NOT EXISTS waterway_important;
+
+CREATE TABLE IF NOT EXISTS waterway_important.changes_z9_z10_z11
+(
+ is_old boolean,
+ id integer,
+ PRIMARY KEY (is_old, id)
+);
+
+CREATE OR REPLACE FUNCTION insert_important_waterway_linestring_gen(full_update bool) RETURNS void AS
+$$
+DECLARE
+ t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+ RAISE LOG 'Refresh waterway z9 z10 z11';
+
+ -- Analyze tracking and source tables before performing update
+ ANALYZE waterway_important.changes_z9_z10_z11;
+ ANALYZE osm_important_waterway_linestring;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_important_waterway_linestring_gen_z11
+ USING waterway_important.changes_z9_z10_z11
+ WHERE full_update IS TRUE OR (
+ waterway_important.changes_z9_z10_z11.is_old IS TRUE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring_gen_z11.id
+ );
+
-- etldoc: osm_important_waterway_linestring -> osm_important_waterway_linestring_gen_z11
INSERT INTO osm_important_waterway_linestring_gen_z11 (geometry, id, name, name_en, name_de, tags)
SELECT ST_Simplify(geometry, ZRes(12)) AS geometry,
@@ -64,9 +193,28 @@ BEGIN
name_de,
tags
FROM osm_important_waterway_linestring
- WHERE
- (update_id IS NULL OR id = update_id) AND
- ST_Length(geometry) > 1000;
+ WHERE (
+ full_update OR
+ EXISTS(
+ SELECT NULL
+ FROM waterway_important.changes_z9_z10_z11
+ WHERE waterway_important.changes_z9_z10_z11.is_old IS FALSE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring.id
+ )
+ ) AND ST_Length(geometry) > 1000
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, name = excluded.name, name_en = excluded.name_en,
+ name_de = excluded.name_de, tags = excluded.tags;
+
+ -- Analyze source table
+ ANALYZE osm_important_waterway_linestring_gen_z11;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_important_waterway_linestring_gen_z10
+ USING waterway_important.changes_z9_z10_z11
+ WHERE full_update IS TRUE OR (
+ waterway_important.changes_z9_z10_z11.is_old IS TRUE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring_gen_z10.id
+ );
-- etldoc: osm_important_waterway_linestring_gen_z11 -> osm_important_waterway_linestring_gen_z10
INSERT INTO osm_important_waterway_linestring_gen_z10 (geometry, id, name, name_en, name_de, tags)
@@ -77,9 +225,28 @@ BEGIN
name_de,
tags
FROM osm_important_waterway_linestring_gen_z11
- WHERE
- (update_id IS NULL OR id = update_id) AND
- ST_Length(geometry) > 4000;
+ WHERE (
+ full_update OR
+ EXISTS(
+ SELECT NULL
+ FROM waterway_important.changes_z9_z10_z11
+ WHERE waterway_important.changes_z9_z10_z11.is_old IS FALSE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring_gen_z11.id
+ )
+ ) AND ST_Length(geometry) > 4000
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, name = excluded.name, name_en = excluded.name_en,
+ name_de = excluded.name_de, tags = excluded.tags;
+
+ -- Analyze source table
+ ANALYZE osm_important_waterway_linestring_gen_z10;
+
+ -- Remove entries which have been deleted from source table
+ DELETE FROM osm_important_waterway_linestring_gen_z9
+ USING waterway_important.changes_z9_z10_z11
+ WHERE full_update IS TRUE OR (
+ waterway_important.changes_z9_z10_z11.is_old IS TRUE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring_gen_z9.id
+ );
-- etldoc: osm_important_waterway_linestring_gen_z10 -> osm_important_waterway_linestring_gen_z9
INSERT INTO osm_important_waterway_linestring_gen_z9 (geometry, id, name, name_en, name_de, tags)
@@ -90,17 +257,39 @@ BEGIN
name_de,
tags
FROM osm_important_waterway_linestring_gen_z10
- WHERE
- (update_id IS NULL OR id = update_id) AND
- ST_Length(geometry) > 8000;
+ WHERE (
+ full_update OR
+ EXISTS(
+ SELECT NULL
+ FROM waterway_important.changes_z9_z10_z11
+ WHERE waterway_important.changes_z9_z10_z11.is_old IS FALSE AND
+ waterway_important.changes_z9_z10_z11.id = osm_important_waterway_linestring_gen_z10.id
+ )
+ ) AND ST_Length(geometry) > 8000
+ ON CONFLICT (id) DO UPDATE SET geometry = excluded.geometry, name = excluded.name, name_en = excluded.name_en,
+ name_de = excluded.name_de, tags = excluded.tags;
+
+ -- noinspection SqlWithoutWhere
+ DELETE FROM waterway_important.changes_z9_z10_z11;
+
+ RAISE LOG 'Refresh waterway z9 z10 z11 done in %', age(clock_timestamp(), t);
END;
$$ LANGUAGE plpgsql;
+-- Ensure tables are emtpy if they haven't been created
TRUNCATE osm_important_waterway_linestring_gen_z11;
TRUNCATE osm_important_waterway_linestring_gen_z10;
TRUNCATE osm_important_waterway_linestring_gen_z9;
-SELECT insert_important_waterway_linestring_gen(NULL);
+SELECT insert_important_waterway_linestring_gen(TRUE);
+
+-- Indexes for queries originating from insert_important_waterway_linestring_gen() function
+CREATE UNIQUE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen_z11_update_idx
+ ON osm_important_waterway_linestring_gen_z11 (id) WHERE ST_Length(geometry) > 4000;
+CREATE UNIQUE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen_z10_update_idx
+ ON osm_important_waterway_linestring_gen_z10 (id) WHERE ST_Length(geometry) > 8000;
+
+-- Geometry Indexes
CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen_z11_geometry_idx
ON osm_important_waterway_linestring_gen_z11 USING gist (geometry);
CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen_z10_geometry_idx
@@ -109,35 +298,57 @@ CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen_z9_geometry_idx
ON osm_important_waterway_linestring_gen_z9 USING gist (geometry);
--- Handle updates
+-- Handle updates on
+-- -- osm_waterway_linestring -> osm_important_waterway_linestring
+-- -- osm_important_waterway_linestring -> osm_important_waterway_linestring_gen_z11
+-- -- osm_important_waterway_linestring -> osm_important_waterway_linestring_gen_z10
+-- -- osm_important_waterway_linestring -> osm_important_waterway_linestring_gen_z9
-CREATE SCHEMA IF NOT EXISTS waterway_important;
+CREATE OR REPLACE AGGREGATE array_cat_agg(anycompatiblearray) (
+ SFUNC=array_cat,
+ STYPE=anycompatiblearray,
+ INITCOND = '{}'
+);
CREATE TABLE IF NOT EXISTS waterway_important.changes
(
- id serial PRIMARY KEY,
osm_id bigint,
is_old boolean,
- name character varying,
- name_en character varying,
- name_de character varying,
- tags hstore
+ PRIMARY KEY (is_old, osm_id)
);
+
+-- Store IDs of changed elements from osm_waterway_linestring table.
CREATE OR REPLACE FUNCTION waterway_important.store() RETURNS trigger AS
$$
BEGIN
IF (tg_op IN ('DELETE', 'UPDATE')) AND OLD.name <> '' AND OLD.waterway = 'river' THEN
- INSERT INTO waterway_important.changes(is_old, name, name_en, name_de, tags)
- VALUES (TRUE, OLD.name, OLD.name_en, OLD.name_de, slice_language_tags(OLD.tags));
+ INSERT INTO waterway_important.changes(is_old, osm_id)
+ VALUES (TRUE, old.osm_id) ON CONFLICT DO NOTHING;
END IF;
IF (tg_op IN ('UPDATE', 'INSERT')) AND NEW.name <> '' AND NEW.waterway = 'river' THEN
- INSERT INTO waterway_important.changes(is_old, name, name_en, name_de, tags)
- VALUES (FALSE, NEW.name, NEW.name_en, NEW.name_de, slice_language_tags(NEW.tags));
+ INSERT INTO waterway_important.changes(is_old, osm_id)
+ VALUES (FALSE, new.osm_id) ON CONFLICT DO NOTHING;
END IF;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
+-- Store IDs of changed elements from osm_important_waterway_linestring table.
+CREATE OR REPLACE FUNCTION waterway_important.important_waterway_linestring_store() RETURNS trigger AS
+$$
+BEGIN
+ IF (tg_op = 'UPDATE' OR tg_op = 'DELETE') THEN
+ INSERT INTO waterway_important.changes_z9_z10_z11 (is_old, id) VALUES (TRUE, old.id) ON CONFLICT DO NOTHING ;
+ END IF;
+
+ IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
+ INSERT INTO waterway_important.changes_z9_z10_z11 (is_old, id) VALUES (FALSE, new.id) ON CONFLICT DO NOTHING;
+ END IF;
+
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
CREATE TABLE IF NOT EXISTS waterway_important.updates
(
id serial PRIMARY KEY,
@@ -161,93 +372,182 @@ BEGIN
-- REFRESH osm_important_waterway_linestring
- -- Compact the change history to keep only the first and last version, and then uniq version of row
- CREATE TEMP TABLE changes_compact AS
- SELECT DISTINCT ON (name, name_en, name_de, tags)
- name,
- name_en,
- name_de,
- tags
- FROM ((
- SELECT DISTINCT ON (osm_id) *
- FROM waterway_important.changes
- WHERE is_old
- ORDER BY osm_id,
- id ASC
- )
- UNION ALL
- (
- SELECT DISTINCT ON (osm_id) *
- FROM waterway_important.changes
- WHERE NOT is_old
- ORDER BY osm_id,
- id DESC
- )) AS t;
+ -- Analyze tracking and source tables before performing update
+ ANALYZE waterway_important.changes;
+ ANALYZE osm_waterway_linestring;
+ -- Fetch updated and deleted Merged-LineString from relation-table filtering for each Merged-LineString which
+ -- contains an updated Source-LineString.
+ -- Additionally attach a list of Source-LineString-IDs to each Merged-LineString in order to unnest them later.
+ CREATE TEMPORARY TABLE affected_merged_linestrings AS
+ SELECT m.id, array_agg(source_id) AS source_ids
+ FROM osm_important_waterway_linestring_source_ids m
+ WHERE EXISTS(
+ SELECT NULL
+ FROM waterway_important.changes c
+ WHERE c.is_old IS TRUE AND c.osm_id = m.source_id
+ )
+ GROUP BY id;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE affected_merged_linestrings;
+
+ -- Delete all Merged-LineStrings which contained an updated or deleted Source-LineString
DELETE
- FROM osm_important_waterway_linestring AS w
- USING changes_compact AS c
- WHERE w.name = c.name
- AND w.name_en IS NOT DISTINCT FROM c.name_en
- AND w.name_de IS NOT DISTINCT FROM c.name_de
- AND w.tags IS NOT DISTINCT FROM c.tags;
+ FROM osm_important_waterway_linestring m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
+ DELETE
+ FROM osm_important_waterway_linestring_source_ids m
+ USING affected_merged_linestrings
+ WHERE affected_merged_linestrings.id = m.id;
- INSERT INTO osm_important_waterway_linestring (geometry, name, name_en, name_de, tags)
- SELECT (ST_Dump(geometry)).geom AS geometry,
- name,
- name_en,
- name_de,
- tags
+ -- Analyze the tables affected by the delete-query in order to speed up subsequent queries
+ ANALYZE osm_important_waterway_linestring;
+ ANALYZE osm_important_waterway_linestring_source_ids;
+
+ -- Create a table containing all LineStrings which should be merged
+ CREATE TEMPORARY TABLE linestrings_to_merge AS
+ -- Add all Source-LineStrings affected by this update
+ SELECT osm_id, NULL::INTEGER AS id, NULL::BIGINT[] AS source_ids, geometry, name, name_en, name_de,
+ slice_language_tags(tags) as tags
+ -- Table containing the IDs of all Source-LineStrings affected by this update
FROM (
- SELECT ST_LineMerge(ST_Union(geometry)) AS geometry,
- w.name,
- w.name_en,
- w.name_de,
- slice_language_tags(w.tags) AS tags
- FROM osm_waterway_linestring AS w
- JOIN changes_compact AS c ON
- w.name = c.name AND w.name_en IS NOT DISTINCT FROM c.name_en AND
- w.name_de IS NOT DISTINCT FROM c.name_de AND
- slice_language_tags(w.tags) IS NOT DISTINCT FROM c.tags
- WHERE w.name <> ''
- AND w.waterway = 'river'
- AND ST_IsValid(geometry)
- GROUP BY w.name, w.name_en, w.name_de, slice_language_tags(w.tags)
- ) AS waterway_union;
+ -- Get Source-LineString-IDs of deleted or updated elements
+ SELECT unnest(affected_merged_linestrings.source_ids)::bigint AS source_id FROM affected_merged_linestrings
+ UNION
+ -- Get Source-LineString-IDs of inserted or updated elements
+ SELECT osm_id AS source_id FROM waterway_important.changes WHERE is_old IS FALSE
+ ORDER BY source_id
+ ) affected_source_linestrings
+ JOIN osm_waterway_linestring ON (
+ affected_source_linestrings.source_id = osm_waterway_linestring.osm_id
+ )
+ WHERE name <> '' AND waterway = 'river' AND ST_IsValid(geometry);
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE affected_merged_linestrings;
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Add all Merged-LineStrings intersecting with Source-LineStrings affected by this update
+ INSERT INTO linestrings_to_merge
+ SELECT NULL::BIGINT AS osm_id, m.id,
+ ARRAY(
+ SELECT s.source_id FROM osm_important_waterway_linestring_source_ids s WHERE s.id = m.id
+ )::BIGINT[] AS source_ids,
+ m.geometry, m.name, m.name_en, m.name_de, m.tags
+ FROM linestrings_to_merge
+ JOIN osm_important_waterway_linestring m ON (ST_Intersects(linestrings_to_merge.geometry, m.geometry));
+
+ -- Analyze the created table to speed up subsequent queries
+ ANALYZE linestrings_to_merge;
+
+ -- Delete all Merged-LineStrings intersecting with Source-LineStrings affected by this update.
+ -- We can use the linestrings_to_merge table since Source-LineStrings affected by this update and present in the
+ -- table will have their ID-Column set to NULL by the previous query.
+ DELETE
+ FROM osm_important_waterway_linestring m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+ DELETE
+ FROM osm_important_waterway_linestring_source_ids m
+ USING linestrings_to_merge
+ WHERE m.id = linestrings_to_merge.id;
+
+ -- Create table containing all LineStrings to and create clusters of intersecting LineStrings partitioned by their
+ -- groups
+ CREATE TEMPORARY TABLE clustered_linestrings_to_merge AS
+ SELECT *,
+ -- Get intersecting clusters by setting minimum distance to 0 and minimum intersecting points to 1.
+ -- https://postgis.net/docs/ST_ClusterDBSCAN.html
+ ST_ClusterDBSCAN(geometry, 0, 1) OVER (PARTITION BY name, name_en, name_de, tags) AS cluster,
+ -- ST_ClusterDBSCAN returns an increasing integer as the cluster-ids within each partition starting at 0.
+ -- This leads to clusters having the same ID across multiple partitions therefore we generate a
+ -- Cluster-Group-ID by utilizing the DENSE_RANK function sorted over the partition columns.
+ DENSE_RANK() OVER (ORDER BY name, name_en, name_de, tags) as cluster_group
+ FROM linestrings_to_merge;
+
+ -- Drop temporary tables early to save resources
+ DROP TABLE linestrings_to_merge;
+
+ -- Create index on cluster columns and analyze the created table to speed up subsequent queries
+ CREATE INDEX ON clustered_linestrings_to_merge (cluster_group, cluster);
+ ANALYZE clustered_linestrings_to_merge;
+
+ -- Create temporary Merged-LineString to Source-LineStrings-ID columns to store relations before they have been
+ -- intersected
+ ALTER TABLE osm_important_waterway_linestring ADD COLUMN IF NOT EXISTS new_source_ids BIGINT[];
+ ALTER TABLE osm_important_waterway_linestring ADD COLUMN IF NOT EXISTS old_source_ids BIGINT[];
+
+ WITH inserted_linestrings AS (
+ -- Merge LineStrings of each cluster and insert them
+ INSERT INTO osm_important_waterway_linestring (geometry, new_source_ids, old_source_ids, name, name_en, name_de,
+ tags)
+ SELECT (ST_Dump(ST_LineMerge(ST_Union(geometry)))).geom AS geometry,
+ -- We use St_Union instead of St_Collect to ensure no overlapping points exist within the geometries
+ -- to merge. https://postgis.net/docs/ST_Union.html
+ -- ST_LineMerge only merges across singular intersections and groups its output into a MultiLineString
+ -- if more than two LineStrings form an intersection or no intersection could be found.
+ -- https://postgis.net/docs/ST_LineMerge.html
+ -- In order to not end up with a mixture of LineStrings and MultiLineStrings we dump eventual
+ -- MultiLineStrings via ST_Dump. https://postgis.net/docs/ST_Dump.html
+ coalesce( array_agg(osm_id) FILTER (WHERE osm_id IS NOT NULL), '{}' )::BIGINT[] AS new_source_ids,
+ array_cat_agg(source_ids)::BIGINT[] as old_source_ids,
+ name,
+ name_en,
+ name_de,
+ tags
+ FROM clustered_linestrings_to_merge
+ GROUP BY cluster_group, cluster, name, name_en, name_de, tags
+ RETURNING id, new_source_ids, old_source_ids, geometry
+ )
+ -- Store OSM-IDs of Source-LineStrings by intersecting Merged-LineStrings with their sources.
+ -- This is required because ST_LineMerge only merges across singular intersections and groups its output into a
+ -- MultiLineString if more than two LineStrings form an intersection or no intersection could be found.
+ INSERT INTO osm_important_waterway_linestring_source_ids (id, source_id)
+ SELECT m.id, source_id
+ FROM (
+ SELECT id, source_id, geometry
+ FROM inserted_linestrings
+ CROSS JOIN LATERAL (
+ SELECT DISTINCT all_source_ids.source_id
+ FROM unnest(
+ array_cat(inserted_linestrings.new_source_ids, inserted_linestrings.old_source_ids)
+ ) AS all_source_ids(source_id)
+ ) source_ids
+ ) m
+ JOIN osm_waterway_linestring s ON (m.source_id = s.osm_id)
+ WHERE ST_Intersects(s.geometry, m.geometry)
+ ON CONFLICT (id, source_id) DO NOTHING;
+
+ -- Cleanup remaining table
+ DROP TABLE clustered_linestrings_to_merge;
+
+ -- Drop temporary Merged-LineString to Source-LineStrings-ID columns
+ ALTER TABLE osm_important_waterway_linestring DROP COLUMN IF EXISTS new_source_ids;
+ ALTER TABLE osm_important_waterway_linestring DROP COLUMN IF EXISTS old_source_ids;
- DROP TABLE changes_compact;
-- noinspection SqlWithoutWhere
DELETE FROM waterway_important.changes;
-- noinspection SqlWithoutWhere
DELETE FROM waterway_important.updates;
RAISE LOG 'Refresh waterway done in %', age(clock_timestamp(), t);
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-CREATE OR REPLACE FUNCTION waterway_important.important_waterway_linestring_gen_refresh() RETURNS trigger AS
-$$
-BEGIN
- IF (tg_op = 'DELETE' OR tg_op = 'UPDATE') THEN
- DELETE FROM osm_important_waterway_linestring_gen_z11 WHERE id = old.id;
- DELETE FROM osm_important_waterway_linestring_gen_z10 WHERE id = old.id;
- DELETE FROM osm_important_waterway_linestring_gen_z9 WHERE id = old.id;
- END IF;
-
- IF (tg_op = 'UPDATE' OR tg_op = 'INSERT') THEN
- PERFORM insert_important_waterway_linestring_gen(new.id);
- END IF;
+ -- Update z11, z10 and z9 tables
+ PERFORM insert_important_waterway_linestring_gen(FALSE);
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
-CREATE TRIGGER trigger_important_waterway_linestring
+CREATE TRIGGER trigger_important_waterway_linestring_store
AFTER INSERT OR UPDATE OR DELETE
ON osm_important_waterway_linestring
FOR EACH ROW
-EXECUTE PROCEDURE waterway_important.important_waterway_linestring_gen_refresh();
+EXECUTE PROCEDURE waterway_important.important_waterway_linestring_store();
CREATE TRIGGER trigger_store
AFTER INSERT OR UPDATE OR DELETE
diff --git a/quickstart.sh b/quickstart.sh
index 3a3ad781..d82b4470 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -75,13 +75,28 @@ echo " : Minimum required Docker version: $MIN_DOCKER_VER+"
echo " : Minimum required docker-compose version: $MIN_COMPOSE_VER+"
echo " : See the .travis build for the currently supported versions."
echo " : Your docker system:"
+
+if ! command -v docker-compose &> /dev/null; then
+ DOCKER_COMPOSE_HYPHEN=false
+else
+ DOCKER_COMPOSE_HYPHEN=true
+fi
+
+function docker_compose_command () {
+ if $DOCKER_COMPOSE_HYPHEN; then
+ docker-compose $@
+ else
+ docker compose $@
+ fi
+}
+
docker --version
-docker-compose --version
+docker_compose_command --version
# based on: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
function version { echo "$@" | tr -d 'v' | tr -cs '0-9.' '.' | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
-COMPOSE_VER=$(docker-compose version --short)
+COMPOSE_VER=$(docker_compose_command version --short)
if [ "$(version "$COMPOSE_VER")" -lt "$(version "$MIN_COMPOSE_VER")" ]; then
echo "ERR: Your Docker-compose version is known to have bugs, please update docker-compose!"
exit 1
@@ -116,7 +131,7 @@ echo " : Started : $STARTDATE "
echo " : Your bash version: $BASH_VERSION"
echo " : Your OS : $OSTYPE"
docker --version
-docker-compose --version
+docker_compose_command --version
if [[ "$OSTYPE" == "linux-gnu" ]]; then
echo " "
@@ -148,6 +163,13 @@ else
echo " "
fi
+# override the output filename based on the area if the default `tiles.mbtiles` is found
+if [[ "$(source .env ; echo "$MBTILES_FILE")" = "tiles.mbtiles" ]]; then
+ MBTILES_FILENAME=${area}.mbtiles
+else
+ MBTILES_FILENAME=$(source .env ; echo "$MBTILES_FILE")
+fi
+
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers"
@@ -165,8 +187,8 @@ make init-dirs
echo " "
echo "-------------------------------------------------------------------------------------"
-echo "====> : Removing old MBTILES if exists ( ./data/${area}.mbtiles ) "
-rm -f "./data/${area}.mbtiles"
+echo "====> : Removing old MBTILES if exists ( ./data/$MBTILES_FILENAME ) "
+rm -f "./data/$MBTILES_FILENAME"
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -277,9 +299,9 @@ fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start generating MBTiles (containing gzipped MVT PBF) using PostGIS. "
-echo " : Output MBTiles: ./data/${area}.mbtiles "
+echo " : Output MBTiles: $MBTILES_FILENAME "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/generate-tiles "
-make generate-tiles-pg
+MBTILES_FILE=$MBTILES_FILENAME make generate-tiles-pg
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -291,8 +313,7 @@ echo "--------------------------------------------------------------------------
echo "====> : Inputs - Outputs md5sum for debugging "
rm -f ./data/quickstart_checklist.chk
{
- find build -type f | sort | xargs md5sum
- find data -type f | sort | xargs md5sum
+ find build data -type f -exec md5sum {} + | sort -k2
} >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
@@ -311,7 +332,7 @@ docker images | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
-echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) "
+echo "====> : (disk space) We have created the new vectortiles ( ./data/$MBTILES_FILENAME ) "
echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
echo " : Data directory content:"
ls -la ./data
diff --git a/tests/import/500_import-highway.osm b/tests/import/500_import-highway.osm
index 8b745512..a3c89157 100644
--- a/tests/import/500_import-highway.osm
+++ b/tests/import/500_import-highway.osm
@@ -165,4 +165,14 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/import/600_import-poi.osm b/tests/import/600_import-poi.osm
index 93de4f6b..17d2ffe8 100644
--- a/tests/import/600_import-poi.osm
+++ b/tests/import/600_import-poi.osm
@@ -28,4 +28,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/test-post-import.sql b/tests/test-post-import.sql
index b1af312c..3a06f5fb 100644
--- a/tests/test-post-import.sql
+++ b/tests/test-post-import.sql
@@ -72,19 +72,24 @@ BEGIN
-- Test 500
-- Verify that road classifications show up at the right zoom level
- SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE highway='motorway';
+ SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE osm_national_network(network);
IF cnt < 1 THEN
- INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 motorway count expected >=1, got ' || cnt);
+ INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 national network count expected >=1, got ' || cnt);
END IF;
- SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE highway='trunk';
+ SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE NOT osm_national_network(network);
IF cnt <> 0 THEN
- INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 trunk count expected 0, got ' || cnt);
+ INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 not national network count expected 0, got ' || cnt);
END IF;
- SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z5 WHERE highway='trunk';
+ SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z5 WHERE highway='motorway';
IF cnt < 1 THEN
- INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z5 trunk count expected >=1, got ' || cnt);
+ INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z5 motorway count expected >=1, got ' || cnt);
+ END IF;
+
+ SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z5 WHERE highway='trunk' AND osm_national_network(network);
+ IF cnt < 1 THEN
+ INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z5 trunk and national network count expected >=1, got ' || cnt);
END IF;
SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z6 WHERE highway='primary';
@@ -161,10 +166,10 @@ BEGIN
END IF;
-- Duplicate route concurrencies collapsed
- SELECT COUNT(*) INTO cnt FROM osm_route_member
+ SELECT COUNT(*) INTO cnt FROM transportation_route_member_coalesced
WHERE network='US:I' AND ref='95';
IF cnt <> 1 THEN
- INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_route_member 1 route membership expected, got ' || cnt);
+ INSERT INTO omt_test_failures VALUES(500, 'import', 'transportation_route_member_coalesced 1 route membership expected, got ' || cnt);
END IF;
-- Test 600
@@ -190,6 +195,20 @@ BEGIN
IF cnt <> 1 THEN
INSERT INTO omt_test_failures VALUES(600, 'import', 'osm_poi_point parcel_locker with name like "OpenMapTiles Parcel Locker PL00%" expected 1, got ' || cnt);
END IF;
+
+ -- verify that charging stations are imported with correct name which can come from tags like brand or operator and can contain ref
+ SELECT COUNT(*) INTO cnt FROM osm_poi_point
+ WHERE subclass = 'charging_station'
+ AND tags->'name' = 'OpenMapTiles Charging Station';
+ IF cnt <> 2 THEN
+ INSERT INTO omt_test_failures VALUES(600, 'import', 'osm_poi_point charging_station with name "OpenMapTiles Charging Station" expected 2, got ' || cnt);
+ END IF;
+ SELECT COUNT(*) INTO cnt FROM osm_poi_polygon
+ WHERE subclass = 'charging_station'
+ AND tags->'name' = 'OpenMapTiles Charging Station Brand';
+ IF cnt <> 1 THEN
+ INSERT INTO omt_test_failures VALUES(600, 'import', 'osm_poi_polygon charging_station with name "OpenMapTiles Charging Station Brand" expected 1, got ' || cnt);
+ END IF;
END;
diff --git a/tests/test-post-update.sql b/tests/test-post-update.sql
index 96953104..d712ac09 100644
--- a/tests/test-post-update.sql
+++ b/tests/test-post-update.sql
@@ -98,7 +98,7 @@ BEGIN
WHERE subclass = 'parcel_locker'
AND tags->'name' like 'OpenMapTiles Parcel Locker%';
IF cnt <> 2 THEN
- INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_point atm with name "OpenMapTiles ATM" expected 2, got ' || cnt);
+ INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_point atm with name "OpenMapTiles Parcel Locker%" expected 2, got ' || cnt);
END IF;
SELECT COUNT(*) INTO cnt FROM osm_poi_point
WHERE subclass = 'parcel_locker'
@@ -107,6 +107,26 @@ BEGIN
INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_point parcel_locker with name "Different operator PL001" expected 1, got ' || cnt);
END IF;
+ -- for charging_station
+ SELECT COUNT(*) INTO cnt FROM osm_poi_point
+ WHERE subclass = 'charging_station'
+ AND tags->'name' = 'OpenMapTiles Charging Station';
+ IF cnt <> 1 THEN
+ INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_point charging_station with name "OpenMapTiles Charging Station" expected 1, got ' || cnt);
+ END IF;
+ SELECT COUNT(*) INTO cnt FROM osm_poi_point
+ WHERE subclass = 'charging_station'
+ AND tags->'name' = 'OpenMapTiles Charging Station Brand';
+ IF cnt <> 1 THEN
+ INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_point charging_station with name "OpenMapTiles Charging Station Brand" expected 1, got ' || cnt);
+ END IF;
+ SELECT COUNT(*) INTO cnt FROM osm_poi_polygon
+ WHERE subclass = 'charging_station'
+ AND tags->'name' = 'OpenMapTiles Charging Station';
+ IF cnt <> 1 THEN
+ INSERT INTO omt_test_failures VALUES(600, 'update', 'osm_poi_polygon charging_station with name "OpenMapTiles Charging Station" expected 1, got ' || cnt);
+ END IF;
+
END;
$$;
diff --git a/tests/update/600_update-poi.osc b/tests/update/600_update-poi.osc
index e03e31f3..2dfed16c 100644
--- a/tests/update/600_update-poi.osc
+++ b/tests/update/600_update-poi.osc
@@ -15,5 +15,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+