diff --git a/.gitignore b/.gitignore index 88239b3..97cf05e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,9 @@ import_queue settings/*.pbf BACKUP-STYLES.sql settings/last.state.txt -settings/timestamp.txt \ No newline at end of file +settings/timestamp.txt +settings/clip/clip.dbf +settings/clip/clip.prj +settings/clip/clip.qpj +settings/clip/clip.shp +settings/clip/clip.shx diff --git a/Makefile b/Makefile index 8e46a54..902ff9f 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,20 @@ ipdb: @echo "------------------------------------------------------------------" @docker inspect $(PROJECT_ID)_db | grep '"IPAddress"' | head -1 | cut -d '"' -f 4 +import_clip: + @echo + @echo "------------------------------------------------------------------" + @echo "Importing clip shapefile" + @echo "------------------------------------------------------------------" + @docker exec -t -i $(PROJECT_ID)_db /usr/bin/shp2pgsql -c -I -D -s 4326 /home/settings/clip/clip.shp | docker exec -i $(PROJECT_ID)_db su - postgres -c "psql gis" + +remove_clip: + @echo + @echo "------------------------------------------------------------------" + @echo "Removing clip shapefile" + @echo "------------------------------------------------------------------" + @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'" + timestamp: @echo @echo "------------------------------------------------------------------" diff --git a/docker-compose.yml b/docker-compose.yml index f043df5..b58bf9f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,8 @@ imposm: - DBSCHEMA_BACKUP=backup # Install some styles if you are using the default mapping. It can be 'yes' or 'no' - QGIS_STYLE=yes + # Use clip in the database + - CLIP=no osmupdate: diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 8aa5582..c6bb946 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -49,11 +49,14 @@ class Importer(object): 'DBSCHEMA_PRODUCTION': 'public', 'DBSCHEMA_IMPORT': 'import', 'DBSCHEMA_BACKUP': 'backup', + 'CLIP': 'no', 'QGIS_STYLE': 'yes' } self.osm_file = None self.mapping_file = None self.post_import_file = None + self.clip_shape_file = None + self.clip_sql_file = None self.qgis_style = None self.cursor = None @@ -82,6 +85,11 @@ class Importer(object): msg = 'SRID not supported : %s' % self.default['SRID'] self.error(msg) + # Check valid CLIP. + if self.default['CLIP'] not in ['yes', 'no']: + msg = 'CLIP not supported : %s' % self.default['CLIP'] + self.error(msg) + # Check valid QGIS_STYLE. if self.default['QGIS_STYLE'] not in ['yes', 'no']: msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE'] @@ -114,6 +122,14 @@ class Importer(object): if f == 'qgis_style.sql': self.qgis_style = join(self.default['SETTINGS'], f) + if f == 'clip': + clip_folder = join(self.default['SETTINGS'], f) + for clip_file in listdir(clip_folder): + if clip_file == 'clip.shp': + self.clip_shape_file = join(clip_folder, clip_file) + if clip_file == 'clip.sql': + self.clip_shape_file = join(clip_folder, clip_file) + if not self.osm_file: msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS'] self.error(msg) @@ -135,6 +151,14 @@ class Importer(object): else: self.info('Not using QGIS default styles.') + if not self.clip_shape_file and self.default['CLIP'] == 'yes': + msg = 'clip.shp is missing and CLIP = yes.' + self.error(msg) + elif self.clip_shape_file and self.default['QGIS_STYLE']: + self.info('%s detected for clipping.' % self.qgis_style) + else: + self.info('No clipping.') + # In docker-compose, we should wait for the DB is ready. self.info('The checkup is OK. The container will continue soon, after the database.') sleep(45) @@ -188,6 +212,26 @@ class Importer(object): command += ['-f', self.qgis_style] call(command) + def _import_clip_function(self): + """Create function clean_tables().""" + command = ['psql'] + command += ['-h', self.default['HOST']] + command += ['-U', self.default['USER']] + command += ['-d', self.default['DATABASE']] + command += ['-f', self.clip_sql_file] + call(command) + + def clip(self): + """Perform clipping if the clip table is here.""" + if self.count_table('clip') == 1: + self.info('Clipping') + command = ['psql'] + command += ['-h', self.default['HOST']] + command += ['-U', self.default['USER']] + command += ['-d', self.default['DATABASE']] + command += ['-c', 'SELECT clean_tables();'] + call(command) + def count_table(self, name): """Check if there is a table starting with name.""" sql = 'select count(*) ' \ @@ -234,6 +278,10 @@ class Importer(object): if self.post_import_file: self.import_custom_sql() + if self.clip_shape_file: + self._import_clip_function() + self.clip() + if self.qgis_style: self.import_qgis_styles() @@ -265,6 +313,9 @@ class Importer(object): database_timestamp = diff.split('.')[0].split('->-')[1] self.update_timestamp(database_timestamp) + if self.clip_shape_file: + self.clip() + self.info('Import diff successful : %s' % diff) else: msg = 'An error occured in imposm with a diff.' diff --git a/settings/clip/clip.sql b/settings/clip/clip.sql new file mode 100644 index 0000000..dc308dd --- /dev/null +++ b/settings/clip/clip.sql @@ -0,0 +1,20 @@ +CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS +$BODY$ +DECLARE osm_tables CURSOR FOR + SELECT table_name + FROM information_schema.tables + WHERE table_schema='public' + AND table_type='BASE TABLE' + AND table_name LIKE 'osm_%'; +BEGIN + FOR osm_table IN osm_tables LOOP + EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN ( + SELECT DISTINCT osm_id + FROM ' || quote_ident(osm_table.table_name) || ' + LEFT JOIN clip ON ST_Intersects(geometry, geom) + WHERE clip.gid IS NULL) + ;'; + END LOOP; +END; +$BODY$ +LANGUAGE plpgsql;