WIP for libraries profile and for making storage container using SouthAfrica as an example

pull/65/head
Tim Sutton 2018-08-07 08:28:17 +02:00
rodzic 82d2982474
commit 4618a1b01f
17 zmienionych plików z 15305 dodań i 0 usunięć

2
.gitignore vendored
Wyświetl plik

@ -15,3 +15,5 @@ settings/clip/clip.prj
settings/clip/clip.qpj
settings/clip/clip.shp
settings/clip/clip.shx
SouthAfrica/south-africa.pbf
Libraries/south-africa.pbf

Wyświetl plik

@ -0,0 +1,11 @@
# This image is intended to be used as a storage
# only container
FROM byrnedo/alpine-curl
MAINTAINER Tim Sutton <tim@kartoza.com>
RUN mkdir /settings
RUN curl -o /settings/country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf
ADD mapping.yml /settings/mapping.yml
ADD clip /settings/clip
ADD qgis_style.sql /settings/qgis_style,sql

140
Libraries/Makefile 100644
Wyświetl plik

@ -0,0 +1,140 @@
PROJECT_ID := libraries
COMPOSE_FILE := docker-compose.yml
# Uncomment the next line if you want to display data with Leaflet.
# COMPOSE_FILE := docker-compose-web.yml
.PHONY: logs
###
# DOCKER MANAGEMENT
###
status:
@echo
@echo "------------------------------------------------------------------"
@echo "Status in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) ps
build:
@echo
@echo "------------------------------------------------------------------"
@echo "Building in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) build
run:
@echo
@echo "------------------------------------------------------------------"
@echo "Running in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up -d --no-recreate
rundev:
@echo
@echo "------------------------------------------------------------------"
@echo "Running in DEVELOPMENT mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up
stop:
@echo
@echo "------------------------------------------------------------------"
@echo "Stopping in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) stop
kill:
@echo
@echo "------------------------------------------------------------------"
@echo "Killing in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) kill
rm: kill
@echo
@echo "------------------------------------------------------------------"
@echo "Removing production instance!!! "
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) rm
rm-volumes:
@echo
@echo "------------------------------------------------------------------"
@echo "Removing all volumes!!!! "
@echo "------------------------------------------------------------------"
@docker volume rm libraries_osm-postgis-data libraries_import_queue libraries_import_done libraries_cache
logs:
@echo
@echo "------------------------------------------------------------------"
@echo "Logs"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs
live_logs:
@echo
@echo "------------------------------------------------------------------"
@echo "Live Logs"
@echo "------------------------------------------------------------------"
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
###
# CLIPPING
###
import_clip:
@echo
@echo "------------------------------------------------------------------"
@echo "Importing clip shapefile"
@echo "------------------------------------------------------------------"
@docker exec -t -i $(PROJECT_ID)_db /usr/bin/shp2pgsql -c -I -D -s 4326 /home/settings/clip/clip.shp | docker exec -i $(PROJECT_ID)_db su - postgres -c "psql gis"
remove_clip:
@echo
@echo "------------------------------------------------------------------"
@echo "Removing clip shapefile"
@echo "------------------------------------------------------------------"
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
###
# STATS
###
timestamp:
@echo
@echo "------------------------------------------------------------------"
@echo "Timestamp"
@echo "------------------------------------------------------------------"
@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
###
# STYLES
###
import_styles: remove_styles
@echo
@echo "------------------------------------------------------------------"
@echo "Importing QGIS styles"
@echo "------------------------------------------------------------------"
@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/qgis_style.sql gis"
remove_styles:
@echo
@echo "------------------------------------------------------------------"
@echo "Removing QGIS styles"
@echo "------------------------------------------------------------------"
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'"
backup_styles:
@echo
@echo "------------------------------------------------------------------"
@echo "Backup QGIS styles to BACKUP.sql"
@echo "------------------------------------------------------------------"
@echo "SET XML OPTION DOCUMENT;" > BACKUP-STYLES.sql
@ docker exec -t $(PROJECT_ID)_db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql

Wyświetl plik

@ -0,0 +1,20 @@
CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
$BODY$
DECLARE osm_tables CURSOR FOR
SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_type='BASE TABLE'
AND table_name LIKE 'osm_%';
BEGIN
FOR osm_table IN osm_tables LOOP
EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
SELECT DISTINCT osm_id
FROM ' || quote_ident(osm_table.table_name) || '
LEFT JOIN clip ON ST_Intersects(geometry, geom)
WHERE clip.gid IS NULL)
;';
END LOOP;
END;
$BODY$
LANGUAGE plpgsql;

Wyświetl plik

@ -0,0 +1,104 @@
version: '2.1'
volumes:
# Make sure that these names are not used in another docker
# compose otherwise this compose that the other will share
# the same data volumes
libraries-postgis-data:
libraries-import-done:
libraries-import-queue:
libraries-cache:
services:
db:
# About the postgresql version, it should match in the dockerfile of docker-imposm3
image: kartoza/postgis:9.6-2.4
hostname: db
environment:
- POSTGRES_USER=docker
- POSTGRES_PASS=docker
- POSTGRES_DBNAME=gis
volumes:
- 'libraries-postgis-data:/var/lib/postgresql'
# Uncomment to use the postgis database from outside the docker network
ports:
- "45432:5432"
healthcheck:
test: "exit 0"
restart: unless-stopped
imposm:
image: kartoza/docker-osm:imposm-latest
#build: docker-imposm3
volumes:
# These are sharable to other containers
- .:/home/settings
- libraries-import-done:/home/import_done
- libraries-import-queue:/home/import_queue
- libraries-cache:/home/cache
depends_on:
db:
condition: service_healthy
environment:
- POSTGRES_USER=docker
- POSTGRES_PASS=docker
- POSTGRES_DBNAME=gis
- POSTGRES_PORT=5432
- POSTGRES_HOST=db
# seconds between 2 executions of the script
# if 0, then no update will be done, only the first initial import from the PBF
- TIME=120
# folder for settings (with *.json and *.sql)
- SETTINGS=settings
# folder for caching
- CACHE=cache
# folder for diff which has been imported
- IMPORT_DONE=import_done
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE=import_queue
# it can be 3857
- SRID=4326
# see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
- OPTIMIZE=false
# see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_PRODUCTION=public
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_IMPORT=import
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_BACKUP=backup
# Install some styles if you are using the default mapping. It can be 'yes' or 'no'
- QGIS_STYLE=yes
# Use clip in the database
- CLIP=no
restart: unless-stopped
osmupdate:
#build: docker-osmupdate
image: kartoza/docker-osm:osmupdate-latest
volumes:
# These are sharable to other containers
- .:/home/settings
- libraries-import-done:/home/import_done
- libraries-import-queue:/home/import_queue
- libraries-cache:/home/cache
environment:
# These are all currently the defaults but listed here for your
# convenience if you want to change them
# the maximum time range to assemble a cumulated changefile.
- MAX_DAYS=100
# osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
- DIFF=sporadic
# argument to determine the maximum number of parallely processed changefiles.
- MAX_MERGE=7
# define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
- COMPRESSION_LEVEL=1
# change the URL to use a custom URL to fetch regional file updates.
- BASE_URL=http://planet.openstreetmap.org/replication/
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE=import_queue
# folder for diff which has been imported
- IMPORT_DONE=import_done
# seconds between 2 executions of the script
# if 0, then no update will be done, only the first initial import from the PBF
- TIME=120
restart: unless-stopped

Wyświetl plik

@ -0,0 +1,3 @@
timestamp=2018-06-21T13\:39\:25Z
sequenceNumber=3024788
replicationUrl=https://planet.openstreetmap.org/replication/minute/

Wyświetl plik

@ -0,0 +1,108 @@
areas:
area_tags: [buildings]
linear_tags: []
generalized_tables:
tables:
amenities:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
amenity:
- library
type: point
buildings:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
building:
- __any__
type: polygon
housenumbers:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
mapping:
addr:housenumber:
- __any__
type: point
housenumbers_interpolated:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
- key: addr:inclusion
name: addr:inclusion
type: string
mapping:
addr:interpolation:
- __any__
type: linestring
landusages:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- name: area
type: webmerc_area
- args:
values:
- library
name: z_order
type: enumerate
mapping:
amenity:
- library
type: polygon

Plik diff jest za duży Load Diff

Plik diff jest za duży Load Diff

Wyświetl plik

@ -0,0 +1 @@
UNDEFINED

Wyświetl plik

@ -0,0 +1,11 @@
# This image is intended to be used as a storage
# only container
FROM byrnedo/alpine-curl
MAINTAINER Tim Sutton <tim@kartoza.com>
RUN mkdir /settings
RUN curl -o /settings/country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf
ADD mapping.yml /settings/mapping.yml
ADD clip /settings/clip
ADD qgis_style.sql /settings/qgis_style,sql

Wyświetl plik

@ -0,0 +1,20 @@
CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
$BODY$
DECLARE osm_tables CURSOR FOR
SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_type='BASE TABLE'
AND table_name LIKE 'osm_%';
BEGIN
FOR osm_table IN osm_tables LOOP
EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
SELECT DISTINCT osm_id
FROM ' || quote_ident(osm_table.table_name) || '
LEFT JOIN clip ON ST_Intersects(geometry, geom)
WHERE clip.gid IS NULL)
;';
END LOOP;
END;
$BODY$
LANGUAGE plpgsql;

Wyświetl plik

@ -0,0 +1,567 @@
areas:
area_tags: [buildings, landuse, leisure, natural, aeroway]
linear_tags: [highway, barrier]
generalized_tables:
landusages_gen0:
source: landusages_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
landusages_gen1:
source: landusages
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
roads_gen0:
source: roads_gen1
tolerance: 200.0
roads_gen1:
source: roads
sql_filter:
type IN (
'motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary',
'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link')
OR class IN('railway')
tolerance: 50.0
waterareas_gen0:
source: waterareas_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
waterareas_gen1:
source: waterareas
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
waterways_gen0:
source: waterways_gen1
tolerance: 200
waterways_gen1:
source: waterways
tolerance: 50.0
tables:
admin:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: admin_level
name: admin_level
type: integer
mapping:
boundary:
- administrative
type: polygon
aeroways:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
aeroway:
- runway
- taxiway
type: linestring
amenities:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
amenity:
- university
- school
- library
- fuel
- hospital
- fire_station
- police
- townhall
type: point
barrierpoints:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
barrier:
- block
- bollard
- cattle_grid
- chain
- cycle_barrier
- entrance
- horse_stile
- gate
- spikes
- lift_gate
- kissing_gate
- fence
- 'yes'
- wire_fence
- toll_booth
- stile
type: point
barrierways:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
barrier:
- city_wall
- fence
- hedge
- retaining_wall
- wall
- bollard
- gate
- spikes
- lift_gate
- kissing_gate
- embankment
- 'yes'
- wire_fence
type: linestring
buildings:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
building:
- __any__
type: polygon
housenumbers:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
mapping:
addr:housenumber:
- __any__
type: point
housenumbers_interpolated:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
- key: addr:inclusion
name: addr:inclusion
type: string
mapping:
addr:interpolation:
- __any__
type: linestring
landusages:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- name: area
type: webmerc_area
- args:
values:
- land
- island
- heath
- railway
- industrial
- commercial
- retail
- residential
- quarry
- zoo
- vineyard
- orchard
- scrub
- hospital
- place_of_worship
- theatre
- cinema
- nature_reserve
- parking
- fuel
- baracks
- library
- college
- school
- university
- golf_course
- allotments
- common
- pitch
- sports_centre
- garden
- recreation_ground
- village_green
- wetland
- grass
- meadow
- wood
- farmland
- farm
- farmyard
- cemetery
- forest
- park
- playground
- footway
- pedestrian
name: z_order
type: enumerate
mapping:
aeroway:
- runway
- taxiway
amenity:
- university
- school
- college
- library
- fuel
- parking
- cinema
- theatre
- place_of_worship
- hospital
barrier:
- hedge
highway:
- pedestrian
- footway
landuse:
- park
- forest
- residential
- retail
- commercial
- industrial
- railway
- cemetery
- grass
- farmyard
- farm
- farmland
- orchard
- vineyard
- wood
- meadow
- village_green
- recreation_ground
- allotments
- quarry
leisure:
- park
- garden
- playground
- golf_course
- sports_centre
- pitch
- stadium
- common
- nature_reserve
man_made:
- pier
military:
- barracks
natural:
- wood
- land
- scrub
- wetland
- heath
place:
- island
tourism:
- zoo
type: polygon
places:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- args:
values:
- locality
- suburb
- hamlet
- village
- town
- city
- county
- region
- state
- country
name: z_order
type: enumerate
- key: population
name: population
type: integer
mapping:
place:
- country
- state
- region
- county
- city
- town
- village
- hamlet
- suburb
- locality
type: point
roads:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- name: type
type: mapping_value
- key: name
name: name
type: string
- key: tunnel
name: tunnel
type: boolint
- key: bridge
name: bridge
type: boolint
- key: oneway
name: oneway
type: direction
- key: ref
name: ref
type: string
- key: layer
name: z_order
type: wayzorder
- key: access
name: access
type: string
- key: service
name: service
type: string
- name: class
type: mapping_key
filters:
reject:
area: ["yes"]
mappings:
railway:
mapping:
railway:
- rail
- tram
- light_rail
- subway
- narrow_gauge
- preserved
- funicular
- monorail
- disused
roads:
mapping:
highway:
- motorway
- motorway_link
- trunk
- trunk_link
- primary
- primary_link
- secondary
- secondary_link
- tertiary
- tertiary_link
- road
- path
- track
- service
- footway
- bridleway
- cycleway
- steps
- pedestrian
- living_street
- unclassified
- residential
- raceway
man_made:
- pier
- groyne
type: linestring
transport_areas:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- apron
railway:
- station
- platform
type: polygon
transport_points:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- key: ref
name: ref
type: string
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- gate
highway:
- motorway_junction
- turning_circle
- bus_stop
railway:
- station
- halt
- tram_stop
- crossing
- level_crossing
- subway_entrance
type: point
waterareas:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
- name: area
type: webmerc_area
mapping:
amenity:
- swimming_pool
landuse:
- basin
- reservoir
leisure:
- swimming_pool
natural:
- water
waterway:
- riverbank
type: polygon
waterways:
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: name
name: name
type: string
- name: type
type: mapping_value
mapping:
barrier:
- ditch
waterway:
- stream
- river
- canal
- drain
- ditch
type: linestring

Plik diff jest za duży Load Diff