kopia lustrzana https://github.com/kartoza/docker-osm
remove clip functions since they are not needed because of the limitto function
admire
rodzic
b139d89a54
commit
9c04f18a1f
45
Makefile
45
Makefile
|
|
@ -80,24 +80,6 @@ live_logs:
|
|||
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
|
||||
|
||||
|
||||
###
|
||||
# CLIPPING
|
||||
###
|
||||
|
||||
|
||||
import_clip:
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Importing clip shapefile into the database"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_imposm /usr/bin/ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nlt PROMOTE_TO_MULTI -f PostgreSQL PG:"host=db user=docker password=docker dbname=gis" /home/settings/clip/clip.shp
|
||||
|
||||
remove_clip:
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Removing clip shapefile from the database"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
|
||||
|
||||
###
|
||||
# STATS
|
||||
|
|
@ -111,33 +93,6 @@ timestamp:
|
|||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
|
||||
|
||||
###
|
||||
# SQL FILES
|
||||
###
|
||||
|
||||
|
||||
import_sql: import_sql
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Importing SQL files"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/clip/clip.sql gis"
|
||||
|
||||
validate_geom: validate_geom
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Validating geom for all tables"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT validate_geom();'"
|
||||
|
||||
clip_tables: clip_tables
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Clip tables using the clip layer"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT clean_tables();'"
|
||||
|
||||
|
||||
|
||||
###
|
||||
# STYLES
|
||||
|
|
|
|||
|
|
@ -48,17 +48,13 @@ class Importer(object):
|
|||
'DBSCHEMA_PRODUCTION': 'public',
|
||||
'DBSCHEMA_IMPORT': 'import',
|
||||
'DBSCHEMA_BACKUP': 'backup',
|
||||
'CLIP': 'no',
|
||||
'QGIS_STYLE': 'yes'
|
||||
}
|
||||
self.osm_file = None
|
||||
self.mapping_file = None
|
||||
self.post_import_file = None
|
||||
self.clip_shape_file = None
|
||||
self.clip_sql_file = None
|
||||
self.clip_json_file = None
|
||||
self.qgis_style = None
|
||||
|
||||
self.cursor = None
|
||||
self.postgis_uri = None
|
||||
|
||||
|
|
@ -90,13 +86,6 @@ class Importer(object):
|
|||
else:
|
||||
self.info('Detect SRID: ' + self.default['SRID'])
|
||||
|
||||
# Check valid CLIP.
|
||||
if self.default['CLIP'] not in ['yes', 'no']:
|
||||
msg = 'CLIP not supported : %s' % self.default['CLIP']
|
||||
self.error(msg)
|
||||
else:
|
||||
self.info('Clip: ' + self.default['CLIP'])
|
||||
|
||||
# Check valid QGIS_STYLE.
|
||||
if self.default['QGIS_STYLE'] not in ['yes', 'no']:
|
||||
msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
|
||||
|
|
@ -138,14 +127,6 @@ class Importer(object):
|
|||
if f == 'qgis_style.sql':
|
||||
self.qgis_style = join(self.default['SETTINGS'], f)
|
||||
|
||||
if f == 'clip':
|
||||
clip_folder = join(self.default['SETTINGS'], f)
|
||||
for clip_file in listdir(clip_folder):
|
||||
if clip_file == 'clip.shp':
|
||||
self.clip_shape_file = join(clip_folder, clip_file)
|
||||
if clip_file == 'clip.sql':
|
||||
self.clip_sql_file = join(clip_folder, clip_file)
|
||||
|
||||
if not self.osm_file:
|
||||
msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
|
||||
self.error(msg)
|
||||
|
|
@ -175,15 +156,6 @@ class Importer(object):
|
|||
else:
|
||||
self.info('Not using QGIS default styles.')
|
||||
|
||||
if not self.clip_shape_file and self.default['CLIP'] == 'yes':
|
||||
msg = 'clip.shp is missing and CLIP = yes.'
|
||||
self.error(msg)
|
||||
elif self.clip_shape_file and self.default['QGIS_STYLE']:
|
||||
self.info('Shapefile for clipping: ' + self.clip_shape_file)
|
||||
self.info('SQL Clipping function: ' + self.clip_sql_file)
|
||||
else:
|
||||
self.info('No *.shp detected in %s, so no clipping.' % self.default['SETTINGS'])
|
||||
|
||||
# In docker-compose, we should wait for the DB is ready.
|
||||
self.info('The checkup is OK.')
|
||||
|
||||
|
|
@ -240,49 +212,8 @@ class Importer(object):
|
|||
command += ['-f', self.qgis_style]
|
||||
call(command)
|
||||
|
||||
def _import_clip_function(self):
|
||||
"""Create function clean_tables().
|
||||
|
||||
The user must import the clip shapefile to the database!
|
||||
"""
|
||||
self.info('Import clip SQL function.')
|
||||
command = ['psql']
|
||||
command += ['-h', self.default['POSTGRES_HOST']]
|
||||
command += ['-U', self.default['POSTGRES_USER']]
|
||||
command += ['-d', self.default['POSTGRES_DBNAME']]
|
||||
command += ['-f', self.clip_sql_file]
|
||||
call(command)
|
||||
self.info('!! If clip file exists it will be imported into the DB!!')
|
||||
|
||||
def perform_clip_in_db(self):
|
||||
"""Perform clipping if the clip table is here."""
|
||||
if self.locate_table('clip') == 1:
|
||||
self.info('Clipping')
|
||||
command = ['psql']
|
||||
command += ['-h', self.default['POSTGRES_HOST']]
|
||||
command += ['-U', self.default['POSTGRES_USER']]
|
||||
command += ['-d', self.default['POSTGRES_DBNAME']]
|
||||
command += ['-c', 'SELECT clean_tables();']
|
||||
call(command)
|
||||
|
||||
def import_clip_in_db(self):
|
||||
"""Import shapefile for clipping"""
|
||||
if not self.clip_shape_file:
|
||||
msg = 'clip.shp is missing'
|
||||
self.error(msg)
|
||||
else:
|
||||
self.info('Loading clip layer into the database')
|
||||
db_connection = "host=%s user=%s password=%s dbname=%s" % (self.default['POSTGRES_HOST'],
|
||||
self.default['POSTGRES_USER'],
|
||||
self.default['POSTGRES_PASS'],
|
||||
self.default['POSTGRES_DBNAME'])
|
||||
clipper = ''' /usr/bin/ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nlt PROMOTE_TO_MULTI \
|
||||
-f PostgreSQL PG:"%s" %s ''' % (db_connection, self.clip_shape_file)
|
||||
print(clipper)
|
||||
call(clipper, shell=True)
|
||||
|
||||
def locate_table(self, name):
|
||||
"""Check if clip table exists in the DB"""
|
||||
"""Check for tables in the DB table exists in the DB"""
|
||||
sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like 'TEMP_TABLE'); """
|
||||
self.cursor.execute(sql.replace('TEMP_TABLE', '%s' % name))
|
||||
# noinspection PyUnboundLocalVariable
|
||||
|
|
@ -305,7 +236,6 @@ class Importer(object):
|
|||
self.info('No more update to the database. Leaving.')
|
||||
|
||||
def _first_pbf_import(self):
|
||||
osm_clip_table = self.locate_table('clip')
|
||||
"""Run the first PBF import into the database."""
|
||||
command = ['imposm', 'import', '-diff', '-deployproduction']
|
||||
command += ['-overwritecache', '-cachedir', self.default['CACHE']]
|
||||
|
|
@ -334,19 +264,10 @@ class Importer(object):
|
|||
if self.post_import_file:
|
||||
self.import_custom_sql()
|
||||
|
||||
if osm_clip_table != 1:
|
||||
self._import_clip_function()
|
||||
if not self.clip_shape_file:
|
||||
msg = 'clip.shp is missing'
|
||||
self.error(msg)
|
||||
else:
|
||||
self.import_clip_in_db()
|
||||
self.perform_clip_in_db()
|
||||
if self.qgis_style:
|
||||
self.import_qgis_styles()
|
||||
|
||||
def _import_diff(self):
|
||||
osm_clip_table = self.locate_table('clip')
|
||||
# Finally launch the listening process.
|
||||
while True:
|
||||
import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
|
||||
|
|
@ -375,9 +296,6 @@ class Importer(object):
|
|||
# Update the timestamp in the file.
|
||||
database_timestamp = diff.split('.')[0].split('->-')[1]
|
||||
self.update_timestamp(database_timestamp)
|
||||
if osm_clip_table == 1:
|
||||
self.perform_clip_in_db()
|
||||
self.info('Import diff successful : %s' % diff)
|
||||
else:
|
||||
msg = 'An error occured in imposm with a diff.'
|
||||
self.error(msg)
|
||||
|
|
|
|||
23
readme.md
23
readme.md
|
|
@ -83,28 +83,19 @@ you don't set a clipping area, you will end with data from all over the world.
|
|||
|
||||
### Clipping
|
||||
|
||||
Imposm support importing limiting the features to be imported using a geojson file.
|
||||
During the initial import or post update imposm uses the flag `-limito` which allows
|
||||
you to define a smaller area that you can work with.
|
||||
This is always desirable to limit the features being imported into the database rather
|
||||
than clipping them.
|
||||
|
||||
**NB:** Ensure you add a geojson covering the area you intent to clip into the clip folder.
|
||||
The geojson can be the same extent of the administrative area of your country or it can be a
|
||||
smaller extent
|
||||
|
||||
You can put a shapefile in the clip folder. This shapefile will be
|
||||
used for clipping every features after the import.
|
||||
This file has to be named 'clip.shp' and in the CRS you are using in the database (4326 by default).
|
||||
The docker compose uses dependency links which ensures that the database is started first before
|
||||
imposm the script now imports the shapefile into the database when the imposm container starts.
|
||||
|
||||
If you add the `clip.shp` file after the containers has started you can still import the
|
||||
shp using the provided makefile with the command:
|
||||
|
||||
`make import_clip`.
|
||||
|
||||
You can remove the clip file : `make remove_clip`.
|
||||
|
||||
**NB:** It is encouraged to simplify the geometry for the `clip.shp` or `clip.geojson` as
|
||||
a simplified geometry is easier to process during the import or clipping. Rather use the
|
||||
minimum bounding box for the area you intent to clip your dataset with.
|
||||
**NB:** It is encouraged to simplify the geometry for the `clip.geojson` as
|
||||
a simplified geometry is easier to process during the import.
|
||||
Rather use the minimum bounding box for the area you intent to clip your dataset with.
|
||||
|
||||
### QGIS Styles
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,38 +0,0 @@
|
|||
CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
|
||||
$BODY$
|
||||
DECLARE osm_tables CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema='public'
|
||||
AND table_type='BASE TABLE'
|
||||
AND table_name LIKE 'osm_%';
|
||||
BEGIN
|
||||
FOR osm_table IN osm_tables LOOP
|
||||
EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
|
||||
SELECT DISTINCT osm_id
|
||||
FROM ' || quote_ident(osm_table.table_name) || '
|
||||
LEFT JOIN clip ON ST_Intersects(geometry, geom) where clip.ogc_fid is NULL)
|
||||
;';
|
||||
END LOOP;
|
||||
END;
|
||||
$BODY$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
-- Function to validate geometry of all tables.
|
||||
-- To run it after creating the function simply run SELECT validate_geom();
|
||||
CREATE OR REPLACE FUNCTION validate_geom() RETURNS void AS
|
||||
$BODY$
|
||||
DECLARE osm_tables CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema='public'
|
||||
AND table_type='BASE TABLE'
|
||||
AND table_name LIKE 'osm_%';
|
||||
BEGIN
|
||||
FOR osm_table IN osm_tables LOOP
|
||||
EXECUTE 'UPDATE ' || quote_ident(osm_table.table_name) || ' SET
|
||||
geometry = ST_MakeValid(geometry) where ST_IsValidReason(geometry) = ''F'' ;';
|
||||
END LOOP;
|
||||
END;
|
||||
$BODY$
|
||||
LANGUAGE plpgsql;
|
||||
Ładowanie…
Reference in New Issue