Modified importer.py to import clip file if it exists and changed logic for finding if tables are missing

docker-imposm3/importer.py

@@ -26,6 +26,7 @@ from os.path import join, exists, abspath, isabs
 from psycopg2 import connect, OperationalError
 from subprocess import call
 from time import sleep
+from pathlib import Path
 
 
 class Importer(object):
@@ -56,6 +57,7 @@ class Importer(object):
         self.post_import_file = None
         self.clip_shape_file = None
         self.clip_sql_file = None
+        self.clip_json_file = None
         self.qgis_style = None
 
         self.cursor = None
@@ -95,7 +97,7 @@ class Importer(object):
             self.error(msg)
         else:
             self.info('Clip: ' + self.default['CLIP'])
-
+            # Check valid CLIP.
         # Check valid QGIS_STYLE.
         if self.default['QGIS_STYLE'] not in ['yes', 'no']:
             msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
@@ -131,6 +133,9 @@ class Importer(object):
             if f == 'post-pbf-import.sql':
                 self.post_import_file = join(self.default['SETTINGS'], f)
 
+            if f == 'clip.geojson':
+                self.clip_json_file = join(self.default['SETTINGS'], f)
+
             if f == 'qgis_style.sql':
                 self.qgis_style = join(self.default['SETTINGS'], f)
 
@@ -158,6 +163,10 @@ class Importer(object):
             self.info('No custom SQL files post-pbf-import.sql detected in %s' % self.default['SETTINGS'])
         else:
             self.info('SQL Post Import: ' + self.post_import_file)
+        if not self.clip_json_file:
+            self.info('No json files to limit import detected in %s' % self.default['SETTINGS'])
+        else:
+            self.info('Geojson Initial Import Clip: ' + self.clip_json_file)
 
         if not self.qgis_style and self.default['QGIS_STYLE'] == 'yes':
             msg = 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % self.default['SETTINGS']
@@ -244,11 +253,11 @@ class Importer(object):
         command += ['-d', self.default['POSTGRES_DBNAME']]
         command += ['-f', self.clip_sql_file]
         call(command)
-        self.info('!! Be sure to run \'make import_clip\' to import the Shapefile into the DB !!')
+        self.info('!! If clip file exists it will be imported into the DB!!')
 
     def perform_clip_in_db(self):
         """Perform clipping if the clip table is here."""
-        if self.count_table('clip') == 1:
+        if self.locate_table('clip') == 1:
             self.info('Clipping')
             command = ['psql']
             command += ['-h', self.default['POSTGRES_HOST']]
@@ -257,19 +266,34 @@ class Importer(object):
             command += ['-c', 'SELECT clean_tables();']
             call(command)
 
-    def count_table(self, name):
+    def import_clip_in_db(self):
-        """Check if there is a table starting with name."""
+        """Import shapefile for clipping"""
-        sql = 'select count(*) ' \
+        if not self.clip_shape_file:
-              'from information_schema.tables ' \
+            msg = 'clip.shp is missing'
-              'where table_name like \'%s\';' % name
+            self.error(msg)
+        else:
+            self.info('Loading clip layer into the database')
+            db_connection = "host=%s user=%s password=%s dbname=%s" % (self.default['POSTGRES_HOST'],
+                                                                       self.default['POSTGRES_USER'],
+                                                                       self.default['POSTGRES_PASS'],
+                                                                       self.default['POSTGRES_DBNAME'])
+            clipper = ''' /usr/bin/ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nlt PROMOTE_TO_MULTI \
+                        -f PostgreSQL PG:"%s" %s ''' % (db_connection, self.clip_shape_file)
+            print(clipper)
+            call(clipper, shell=True)
+
+    def locate_table(self, name):
+        """Check if clip table exists in the DB"""
+        sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like  'TEMP_TABLE'); """
+        self.cursor.execute(sql.replace('TEMP_TABLE', '%s' % name))
         # noinspection PyUnboundLocalVariable
-        self.cursor.execute(sql)
         return self.cursor.fetchone()[0]
 
     def run(self):
         """First checker."""
-        osm_tables = self.count_table('osm_%')
+        osm_tables = self.locate_table('osm_%')
-        if osm_tables < 1:
+
+        if osm_tables != 1:
             # It means that the DB is empty. Let's import the PBF file.
             self._first_pbf_import()
         else:
@@ -282,6 +306,7 @@ class Importer(object):
             self.info('No more update to the database. Leaving.')
 
     def _first_pbf_import(self):
+        osm_clip_table = self.locate_table('clip')
         """Run the first PBF import into the database."""
         command = ['imposm', 'import', '-diff', '-deployproduction']
         command += ['-overwritecache', '-cachedir', self.default['CACHE']]
@@ -293,6 +318,7 @@ class Importer(object):
         command += ['-diffdir', self.default['SETTINGS']]
         command += ['-mapping', self.mapping_file]
         command += ['-read', self.osm_file]
+        command += ['-limitto', self.clip_json_file]
         command += ['-write', '-connection', self.postgis_uri]
         self.info('The database is empty. Let\'s import the PBF : %s' % self.osm_file)
         self.info(' '.join(command))
@@ -309,14 +335,15 @@ class Importer(object):
         if self.post_import_file:
             self.import_custom_sql()
 
-        if self.clip_shape_file:
+        if osm_clip_table != 1:
             self._import_clip_function()
+            self.import_clip_in_db()
             self.perform_clip_in_db()
-
         if self.qgis_style:
             self.import_qgis_styles()
 
     def _import_diff(self):
+        osm_clip_table = self.locate_table('clip')
         # Finally launch the listening process.
         while True:
             import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
@@ -331,6 +358,7 @@ class Importer(object):
                     command += ['-srid', self.default['SRID']]
                     command += ['-diffdir', self.default['SETTINGS']]
                     command += ['-mapping', self.mapping_file]
+                    command += ['-limitto', self.clip_json_file]
                     command += ['-connection', self.postgis_uri]
                     command += [join(self.default['IMPORT_QUEUE'], diff)]
 
@@ -344,10 +372,8 @@ class Importer(object):
                         # Update the timestamp in the file.
                         database_timestamp = diff.split('.')[0].split('->-')[1]
                         self.update_timestamp(database_timestamp)
-
+                        if osm_clip_table == 1:
-                        if self.clip_shape_file:
                             self.perform_clip_in_db()
-
                         self.info('Import diff successful : %s' % diff)
                     else:
                         msg = 'An error occured in imposm with a diff.'

settings/clip.geojson

@@ -0,0 +1,8 @@
+{
+"type": "FeatureCollection",
+"name": "clip",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { "ID_0": 126, "ISO": "LSO", "NAME_0": "Lesotho", "OBJECTID_1": 135, "ISO3": "LSO", "NAME_ENGLI": "Lesotho", "NAME_ISO": "LESOTHO", "NAME_FAO": "Lesotho", "NAME_LOCAL": "Lesotho", "NAME_OBSOL": null, "NAME_VARIA": "Basutoland", "NAME_NONLA": null, "NAME_FRENC": "Lesotho", "NAME_SPANI": "Lesotho", "NAME_RUSSI": "Лесото", "NAME_ARABI": "ليسوتو", "NAME_CHINE": "莱索托", "WASPARTOF": null, "CONTAINS": null, "SOVEREIGN": "Lesotho", "ISO2": "LS", "WWW": null, "FIPS": "LT", "ISON": 426.0, "VALIDFR": "19661004", "VALIDTO": "Present", "POP2000": 2034667.0, "SQKM": 30351.641, "POPSQKM": 67.036474238699995, "UNREGION1": "Southern Africa", "UNREGION2": "Africa", "DEVELOPING": 1.0, "CIS": 0.0, "Transition": 0.0, "OECD": 0.0, "WBREGION": "Sub-Saharan Africa", "WBINCOME": "Low income", "WBDEBT": "Less indebted", "WBOTHER": null, "CEEAC": 0.0, "CEMAC": 0.0, "CEPLG": 0.0, "COMESA": 0.0, "EAC": 0.0, "ECOWAS": 0.0, "IGAD": 0.0, "IOC": 0.0, "MRU": 0.0, "SACU": 1.0, "UEMOA": 0.0, "UMA": 0.0, "PALOP": 0.0, "PARTA": 0.0, "CACM": 0.0, "EurAsEC": 0.0, "Agadir": 0.0, "SAARC": 0.0, "ASEAN": 0.0, "NAFTA": 0.0, "GCC": 0.0, "CSN": 0.0, "CARICOM": 0.0, "EU": 0.0, "CAN": 0.0, "ACP": 1.0, "Landlocked": 1.0, "AOSIS": 0.0, "SIDS": 0.0, "Islands": 0.0, "LDC": 1.0, "width": 2.436689376830941, "height": 2.0969009399414631, "area": 5.1094962446221786, "perimeter": 9.0671806335448082 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 27.029069900512752, -30.668960571289062 ], [ 29.465759277343693, -30.668960571289062 ], [ 29.465759277343693, -28.572059631347599 ], [ 27.029069900512752, -28.572059631347599 ], [ 27.029069900512752, -30.668960571289062 ] ] ] } }
+]
+}