diff --git a/.gitignore b/.gitignore index d0244ccd..23fd4d18 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,8 @@ quickstart.log # imput / output data data/* +# input wikidata +wikidata/* + # generated source files build/* \ No newline at end of file diff --git a/Makefile b/Makefile index 4e400dab..1c2ab13b 100644 --- a/Makefile +++ b/Makefile @@ -126,6 +126,9 @@ list: download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh +download-wikidata: + mkdir -p wikidata && docker-compose run --rm --entrypoint /usr/src/app/download-gz.sh import-wikidata + start-tileserver: @echo " " @echo "***********************************************************" diff --git a/docker-compose.yml b/docker-compose.yml index c7cf7efe..8d192bd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,14 +68,22 @@ services: - ./build:/mapping - cache:/cache import-sql: - image: "openmaptiles/import-sql:0.6" + image: "openmaptiles/import-sql:0.7" env_file: .env networks: - postgres_conn volumes: - ./build:/sql + import-wikidata: + image: "openmaptiles/import-wikidata:0.1" + env_file: .env + command: import-wikidata + networks: + - postgres_conn + volumes: + - ./wikidata:/import openmaptiles-tools: - image: "openmaptiles/openmaptiles-tools:0.7" + image: "openmaptiles/openmaptiles-tools:0.8" env_file: .env networks: - postgres_conn diff --git a/layers/aerodrome_label/update.sql b/layers/aerodrome_label/update.sql index bbf9d9b6..541d7bae 100644 --- a/layers/aerodrome_label/update.sql +++ b/layers/aerodrome_label/update.sql @@ -9,7 +9,7 @@ BEGIN WHERE ST_GeometryType(geometry) <> 'ST_Point'; UPDATE osm_aerodrome_label_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; $$ LANGUAGE plpgsql; diff --git a/layers/mountain_peak/update_peak_point.sql b/layers/mountain_peak/update_peak_point.sql index 466c9cb5..666bf971 100644 --- a/layers/mountain_peak/update_peak_point.sql +++ b/layers/mountain_peak/update_peak_point.sql @@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON mountain_peak_point.updates; CREATE OR REPLACE FUNCTION update_osm_peak_point() RETURNS VOID AS $$ BEGIN UPDATE osm_peak_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/place/update_city_point.sql b/layers/place/update_city_point.sql index 11f72c4c..e7fa401b 100644 --- a/layers/place/update_city_point.sql +++ b/layers/place/update_city_point.sql @@ -41,7 +41,7 @@ BEGIN WHERE osm.osm_id = ne.osm_id; UPDATE osm_city_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/place/update_continent_point.sql b/layers/place/update_continent_point.sql index 99bc2888..9e48c54e 100644 --- a/layers/place/update_continent_point.sql +++ b/layers/place/update_continent_point.sql @@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON place_continent_point.updates; CREATE OR REPLACE FUNCTION update_osm_continent_point() RETURNS VOID AS $$ BEGIN UPDATE osm_continent_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/place/update_country_point.sql b/layers/place/update_country_point.sql index d6a88552..7a16ff0d 100644 --- a/layers/place/update_country_point.sql +++ b/layers/place/update_country_point.sql @@ -37,7 +37,7 @@ BEGIN WHERE "rank" = 0; UPDATE osm_country_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/place/update_island_point.sql b/layers/place/update_island_point.sql index 2e781e1d..cedb6468 100644 --- a/layers/place/update_island_point.sql +++ b/layers/place/update_island_point.sql @@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON place_island_point.updates; CREATE OR REPLACE FUNCTION update_osm_island_point() RETURNS VOID AS $$ BEGIN UPDATE osm_island_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/place/update_island_polygon.sql b/layers/place/update_island_polygon.sql index 64b18bf9..29eebd3b 100644 --- a/layers/place/update_island_polygon.sql +++ b/layers/place/update_island_polygon.sql @@ -7,7 +7,7 @@ BEGIN UPDATE osm_island_polygon SET geometry=ST_PointOnSurface(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; UPDATE osm_island_polygon - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; ANALYZE osm_island_polygon; diff --git a/layers/place/update_state_point.sql b/layers/place/update_state_point.sql index ff1f0497..c2ab23a5 100644 --- a/layers/place/update_state_point.sql +++ b/layers/place/update_state_point.sql @@ -33,7 +33,7 @@ BEGIN DELETE FROM osm_state_point WHERE "rank" IS NULL; UPDATE osm_state_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql index c9c413b2..c7e1543d 100644 --- a/layers/poi/update_poi_point.sql +++ b/layers/poi/update_poi_point.sql @@ -13,7 +13,7 @@ BEGIN WHERE funicular = 'yes' and subclass='station'; UPDATE osm_poi_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/poi/update_poi_polygon.sql b/layers/poi/update_poi_polygon.sql index ec56f947..ff6061d4 100644 --- a/layers/poi/update_poi_polygon.sql +++ b/layers/poi/update_poi_polygon.sql @@ -22,7 +22,7 @@ BEGIN WHERE funicular = 'yes' and subclass='station'; UPDATE osm_poi_polygon - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; ANALYZE osm_poi_polygon; diff --git a/layers/water_name/update_marine_point.sql b/layers/water_name/update_marine_point.sql index 8cfe0e16..b42133c6 100644 --- a/layers/water_name/update_marine_point.sql +++ b/layers/water_name/update_marine_point.sql @@ -22,7 +22,7 @@ BEGIN WHERE osm.osm_id = ne.osm_id; UPDATE osm_marine_point - SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry) + SET tags = update_tags(tags, geometry) WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL; END; diff --git a/layers/water_name/update_water_lakeline.sql b/layers/water_name/update_water_lakeline.sql index 3ca5ea28..ff7b25a0 100644 --- a/layers/water_name/update_water_lakeline.sql +++ b/layers/water_name/update_water_lakeline.sql @@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS ( SELECT wp.osm_id, ll.wkb_geometry AS geometry, name, name_en, name_de, - delete_empty_keys(tags) || get_basic_names(tags, ll.wkb_geometry) AS tags, + update_tags(tags, ll.wkb_geometry) AS tags, ST_Area(wp.geometry) AS area FROM osm_water_polygon AS wp INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id diff --git a/layers/water_name/update_water_point.sql b/layers/water_name/update_water_point.sql index db6c614b..88742010 100644 --- a/layers/water_name/update_water_point.sql +++ b/layers/water_name/update_water_point.sql @@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( SELECT wp.osm_id, ST_PointOnSurface(wp.geometry) AS geometry, wp.name, wp.name_en, wp.name_de, - delete_empty_keys(wp.tags) || get_basic_names(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags, + update_tags(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags, ST_Area(wp.geometry) AS area FROM osm_water_polygon AS wp LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id diff --git a/layers/waterway/update_waterway_linestring.sql b/layers/waterway/update_waterway_linestring.sql index 8463e33f..e94c8c5c 100644 --- a/layers/waterway/update_waterway_linestring.sql +++ b/layers/waterway/update_waterway_linestring.sql @@ -3,10 +3,17 @@ DROP TRIGGER IF EXISTS trigger_refresh ON osm_waterway_linestring; DO $$ BEGIN - update osm_waterway_linestring SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry); - update osm_waterway_linestring_gen1 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry); - update osm_waterway_linestring_gen2 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry); - update osm_waterway_linestring_gen3 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry); + update osm_waterway_linestring + SET tags = update_tags(tags, geometry); + + update osm_waterway_linestring_gen1 + SET tags = update_tags(tags, geometry); + + update osm_waterway_linestring_gen2 + SET tags = update_tags(tags, geometry); + + update osm_waterway_linestring_gen3 + SET tags = update_tags(tags, geometry); END $$; @@ -17,7 +24,7 @@ CREATE OR REPLACE FUNCTION waterway_linestring.refresh() RETURNS trigger AS $BODY$ BEGIN -- RAISE NOTICE 'Refresh waterway_linestring %', NEW.osm_id; - NEW.tags = delete_empty_keys(NEW.tags) || get_basic_names(NEW.tags, NEW.geometry); + NEW.tags = update_tags(NEW.tags, NEW.geometry); RETURN NEW; END; $BODY$ diff --git a/quickstart.sh b/quickstart.sh index 5ce20901..58ac0a74 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -129,6 +129,7 @@ echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata mkdir -p pgdata mkdir -p build mkdir -p data +mkdir -p wikidata echo " " echo "-------------------------------------------------------------------------------------" @@ -231,6 +232,14 @@ echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copy echo " : Thank you OpenStreetMap Contributors ! " docker-compose run --rm import-osm +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing Wikidata: ./wikidata/latest-all.json.gz -> PostgreSQL" +echo " : Source code: https://github.com/openmaptiles/import-wikidata " +echo " : The Wikidata license: https://www.wikidata.org/wiki/Wikidata:Database_download/en#License " +echo " : Thank you Wikidata Contributors ! " +docker-compose run --rm import-wikidata + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL "