diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 36bb627..dffd5e5 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -56,6 +56,7 @@ class Importer(object): self.post_import_file = None self.clip_json_file = None self.qgis_style = None + self.clip_sql_file = None self.cursor = None self.postgis_uri = None @@ -133,6 +134,9 @@ class Importer(object): if f == 'qgis_style.sql': self.qgis_style = join(self.default['SETTINGS'], f) + if f == 'clip.sql': + self.clip_sql_file = join(self.default['SETTINGS'], f) + if not self.osm_file: msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS'] self.error(msg) @@ -149,6 +153,7 @@ class Importer(object): self.info('No custom SQL files post-pbf-import.sql detected in %s' % self.default['SETTINGS']) else: self.info('SQL Post Import: ' + self.post_import_file) + if not self.clip_json_file: self.info('No json files to limit import detected in %s' % self.default['SETTINGS']) else: @@ -170,6 +175,14 @@ class Importer(object): else: self.info('No *.geojson detected, so no clipping.') + if not self.clip_sql_file and self.default['CLIP'] == 'yes': + msg = 'SQL for clipping is missing and CLIP = yes.' + self.error(msg) + elif self.clip_sql_file and self.default['QGIS_STYLE']: + self.info('SQL for clipping: ' + self.clip_sql_file) + else: + self.info('No *.sql detected, so no clipping.') + # In docker-compose, we should wait for the DB is ready. self.info('The checkup is OK.') @@ -226,6 +239,23 @@ class Importer(object): command += ['-f', self.qgis_style] call(command) + def _import_clip_function(self): + """Create function clean_tables(). + + The user must import the clip shapefile to the database! + """ + self.info('Import clip SQL function.') + command = ['psql'] + command += ['-h', self.default['POSTGRES_HOST']] + command += ['-U', self.default['POSTGRES_USER']] + command += ['-d', self.default['POSTGRES_DBNAME']] + command += ['-f', self.clip_sql_file] + call(command) + + def perform_cron(self): + cron_sql = """ SELECT cron.schedule('*/59 * * * *', $$select clean_tables()$$); """ + self.cursor.execute(cron_sql) + def locate_table(self, name): """Check for tables in the DB table exists in the DB""" sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like 'TEMP_TABLE'); """ @@ -233,6 +263,16 @@ class Importer(object): # noinspection PyUnboundLocalVariable return self.cursor.fetchone()[0] + def clip_importer(self): + clipper = ''' ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nln clip -nlt PROMOTE_TO_MULTI \ + -f PostgreSQL PG:"host=%s user=%s password=%s dbname=%s port=5432" %s ''' % (self.default['POSTGRES_HOST'], + self.default['POSTGRES_USER'], + self.default['POSTGRES_PASS'], + self.default['POSTGRES_DBNAME'], + self.clip_json_file) + + call(clipper, shell=True) + def run(self): """First checker.""" osm_tables = self.locate_table('osm_%') @@ -249,10 +289,7 @@ class Importer(object): 'The database is not empty. Let\'s import only diff files.') if self.default['TIME'] != '0': - if self.clip_json_file: - self._import_diff(['-limitto', self.clip_json_file]) - else: - self._import_diff([]) + self._import_diff() else: self.info('No more update to the database. Leaving.') @@ -288,7 +325,15 @@ class Importer(object): if self.qgis_style: self.import_qgis_styles() - def _import_diff(self, args): + if self.clip_sql_file: + self._import_clip_function() + + clip_tables = self.locate_table('clip') + if clip_tables != 1 and self.clip_json_file: + self.clip_importer() + self.perform_cron() + + def _import_diff(self): # Finally launch the listening process. while True: import_queue = sorted(listdir(self.default['IMPORT_QUEUE'])) @@ -306,7 +351,7 @@ class Importer(object): command += ['-connection', self.postgis_uri] command += [join(self.default['IMPORT_QUEUE'], diff)] - self.info(command.extend(args)) + self.info(' '.join(command)) if call(command) == 0: move( join(self.default['IMPORT_QUEUE'], diff), diff --git a/docker-osmenrich/Dockerfile b/docker-osmenrich/Dockerfile index 43dd6a1..60d0950 100644 --- a/docker-osmenrich/Dockerfile +++ b/docker-osmenrich/Dockerfile @@ -6,6 +6,10 @@ RUN pip3 install -r /home/requirements.txt ADD enrich.py /home/ +RUN if [ ! -f /home/check_tables.sql ]; then \ + echo "SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like 'osm_%');" >> /home/check_tables.sql;\ + fi; + WORKDIR /home CMD ["python3", "-u", "/home/enrich.py"] diff --git a/settings/clip.sql b/settings/clip.sql new file mode 100644 index 0000000..d834353 --- /dev/null +++ b/settings/clip.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS +$BODY$ +DECLARE osm_tables CURSOR FOR + SELECT table_name + FROM information_schema.tables + WHERE table_schema='public' + AND table_type='BASE TABLE' + AND table_name LIKE 'osm_%'; +BEGIN + FOR osm_table IN osm_tables LOOP + EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN ( + SELECT DISTINCT osm_id + FROM ' || quote_ident(osm_table.table_name) || ' + LEFT JOIN clip ON ST_Intersects(geometry, geom) where clip.ogc_fid is NULL) + ;'; + END LOOP; +END; +$BODY$ +LANGUAGE plpgsql; + +-- Function to validate geometry of all tables. +-- To run it after creating the function simply run SELECT validate_geom(); +CREATE OR REPLACE FUNCTION validate_geom() RETURNS void AS +$BODY$ +DECLARE osm_tables CURSOR FOR + SELECT table_name + FROM information_schema.tables + WHERE table_schema='public' + AND table_type='BASE TABLE' + AND table_name LIKE 'osm_%'; +BEGIN + FOR osm_table IN osm_tables LOOP + EXECUTE 'UPDATE ' || quote_ident(osm_table.table_name) || ' SET + geometry = ST_MakeValid(geometry) where ST_IsValidReason(geometry) = ''F'' ;'; + END LOOP; +END; +$BODY$ +LANGUAGE plpgsql;