Fix clipping issues by using limito and refactor some code (#87)

* Fix import error due to differences in geom type

* fix id missing in clip table and skipping importing shapefiles with geometry errors

* fix clip id error in the sql

* Change postgresql version from 9.6 to 11

* Added sql to validate geometry and other make commands to run them manually

* added pgwatch to docker-compose

* Modified importer.py to import clip file if it exists and changed logic for finding if tables are missing

* updated readme file

* fix logic on first run for importing data

* small changes to revert original logic

* remove clip functions since they are not needed because of the limitto function

* Added logic to check if the geojson should be used if it exist or not, added martin to server vector tiles from the DB

* update readme

Makefile

@@ -80,24 +80,6 @@ live_logs:
 	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
 
 
-###
-#    CLIPPING
-###
-
-
-import_clip:
-	@echo
-	@echo "------------------------------------------------------------------"
-	@echo "Importing clip shapefile into the database"
-	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_imposm /usr/bin/ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nlt PROMOTE_TO_MULTI -f PostgreSQL PG:"host=db user=docker password=docker dbname=gis" /home/settings/clip/clip.shp
-
-remove_clip:
-	@echo
-	@echo "------------------------------------------------------------------"
-	@echo "Removing clip shapefile from the database"
-	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
 
 ###
 #    STATS
@@ -111,33 +93,6 @@ timestamp:
 	@echo "------------------------------------------------------------------"
 	@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
 
-###
-#    SQL FILES
-###
-
-
-import_sql: import_sql
-	@echo
-	@echo "------------------------------------------------------------------"
-	@echo "Importing SQL files"
-	@echo "------------------------------------------------------------------"
-	@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/clip/clip.sql gis"
-
-validate_geom: validate_geom
-	@echo
-	@echo "------------------------------------------------------------------"
-	@echo "Validating geom for all tables"
-	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT validate_geom();'"
-
-clip_tables: clip_tables
-	@echo
-	@echo "------------------------------------------------------------------"
-	@echo "Clip tables using the clip layer"
-	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT clean_tables();'"
-
-
 
 ###
 #    STYLES

docker-compose-web.yml

@@ -6,15 +6,37 @@ version: '2.1'
 
 services:
   qgisserver:
-    image: kartoza/qgis-server:2.18
+    image: camptocamp/qgis-server:3.6
     hostname: dockerosm_qgisserver
     container_name: dockerosm_qgisserver
+    environment:
+      - QGIS_PROJECT_FILE=/project/project.qgs
+      - GIS_SERVER_LOG_LEVEL=DEBUG
+      - MAX_REQUESTS_PER_PROCESS=100
     volumes:
       - ./logs:/var/log/apache2
       - ./web:/project
       - ./settings:/web/settings
+    depends_on:
+      db:
+        condition: service_healthy
     links:
       - db:db
     ports:
       - 8198:80
-    restart: unless-stopped
+    restart: on-failure
+
+  # Server vector tiles from PostgreSQL DB
+  martin:
+    image: urbica/martin
+    hostname: dockerosm_martin
+    container_name: dockerosm_martin
+    restart: on-failure
+    ports:
+      - 3000:3000
+    environment:
+      - WATCH_MODE=true
+      - DATABASE_URL=postgres://docker:docker@db/gis
+    depends_on:
+      db:
+        condition: service_healthy

docker-imposm3/importer.py

@@ -54,10 +54,8 @@ class Importer(object):
         self.osm_file = None
         self.mapping_file = None
         self.post_import_file = None
-        self.clip_shape_file = None
+        self.clip_json_file = None
-        self.clip_sql_file = None
         self.qgis_style = None
-
         self.cursor = None
         self.postgis_uri = None
 
@@ -88,14 +86,12 @@ class Importer(object):
             self.error(msg)
         else:
             self.info('Detect SRID: ' + self.default['SRID'])
-
             # Check valid CLIP.
         if self.default['CLIP'] not in ['yes', 'no']:
             msg = 'CLIP not supported : %s' % self.default['CLIP']
             self.error(msg)
         else:
             self.info('Clip: ' + self.default['CLIP'])
-
         # Check valid QGIS_STYLE.
         if self.default['QGIS_STYLE'] not in ['yes', 'no']:
             msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
@@ -131,17 +127,12 @@ class Importer(object):
             if f == 'post-pbf-import.sql':
                 self.post_import_file = join(self.default['SETTINGS'], f)
 
+            if f == 'clip.geojson':
+                self.clip_json_file = join(self.default['SETTINGS'], f)
+
             if f == 'qgis_style.sql':
                 self.qgis_style = join(self.default['SETTINGS'], f)
 
-            if f == 'clip':
-                clip_folder = join(self.default['SETTINGS'], f)
-                for clip_file in listdir(clip_folder):
-                    if clip_file == 'clip.shp':
-                        self.clip_shape_file = join(clip_folder, clip_file)
-                    if clip_file == 'clip.sql':
-                        self.clip_sql_file = join(clip_folder, clip_file)
-
         if not self.osm_file:
             msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
             self.error(msg)
@@ -158,6 +149,10 @@ class Importer(object):
             self.info('No custom SQL files post-pbf-import.sql detected in %s' % self.default['SETTINGS'])
         else:
             self.info('SQL Post Import: ' + self.post_import_file)
+        if not self.clip_json_file:
+            self.info('No json files to limit import detected in %s' % self.default['SETTINGS'])
+        else:
+            self.info('Geojson Initial Import Clip: ' + self.clip_json_file)
 
         if not self.qgis_style and self.default['QGIS_STYLE'] == 'yes':
             msg = 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % self.default['SETTINGS']
@@ -167,14 +162,13 @@ class Importer(object):
         else:
             self.info('Not using QGIS default styles.')
 
-        if not self.clip_shape_file and self.default['CLIP'] == 'yes':
+        if not self.clip_json_file and self.default['CLIP'] == 'yes':
-            msg = 'clip.shp is missing and CLIP = yes.'
+            msg = 'clip.geojson is missing and CLIP = yes.'
             self.error(msg)
-        elif self.clip_shape_file and self.default['QGIS_STYLE']:
+        elif self.clip_json_file and self.default['QGIS_STYLE']:
-            self.info('Shapefile for clipping: ' + self.clip_shape_file)
+            self.info('Geojson for clipping: ' + self.clip_json_file)
-            self.info('SQL Clipping function: ' + self.clip_sql_file)
         else:
-            self.info('No *.shp detected in %s, so no clipping.' % self.default['SETTINGS'])
+            self.info('No *.geojson detected, so no clipping.')
 
         # In docker-compose, we should wait for the DB is ready.
         self.info('The checkup is OK.')
@@ -232,56 +226,37 @@ class Importer(object):
         command += ['-f', self.qgis_style]
         call(command)
 
-    def _import_clip_function(self):
+    def locate_table(self, name):
-        """Create function clean_tables().
+        """Check for tables in the DB table exists in the DB"""
-
+        sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like  'TEMP_TABLE'); """
-        The user must import the clip shapefile to the database!
+        self.cursor.execute(sql.replace('TEMP_TABLE', '%s' % name))
-        """
-        self.info('Import clip SQL function.')
-        command = ['psql']
-        command += ['-h', self.default['POSTGRES_HOST']]
-        command += ['-U', self.default['POSTGRES_USER']]
-        command += ['-d', self.default['POSTGRES_DBNAME']]
-        command += ['-f', self.clip_sql_file]
-        call(command)
-        self.info('!! Be sure to run \'make import_clip\' to import the Shapefile into the DB !!')
-
-    def perform_clip_in_db(self):
-        """Perform clipping if the clip table is here."""
-        if self.count_table('clip') == 1:
-            self.info('Clipping')
-            command = ['psql']
-            command += ['-h', self.default['POSTGRES_HOST']]
-            command += ['-U', self.default['POSTGRES_USER']]
-            command += ['-d', self.default['POSTGRES_DBNAME']]
-            command += ['-c', 'SELECT clean_tables();']
-            call(command)
-
-    def count_table(self, name):
-        """Check if there is a table starting with name."""
-        sql = 'select count(*) ' \
-              'from information_schema.tables ' \
-              'where table_name like \'%s\';' % name
         # noinspection PyUnboundLocalVariable
-        self.cursor.execute(sql)
         return self.cursor.fetchone()[0]
 
     def run(self):
         """First checker."""
-        osm_tables = self.count_table('osm_%')
+        osm_tables = self.locate_table('osm_%')
-        if osm_tables < 1:
+
+        if osm_tables != 1:
             # It means that the DB is empty. Let's import the PBF file.
-            self._first_pbf_import()
+
+            if self.clip_json_file:
+                self._first_pbf_import(['-limitto', self.clip_json_file])
+            else:
+                self._first_pbf_import([])
         else:
             self.info(
                 'The database is not empty. Let\'s import only diff files.')
 
         if self.default['TIME'] != '0':
-            self._import_diff()
+            if self.clip_json_file:
+                self._import_diff(['-limitto', self.clip_json_file])
+            else:
+                self._import_diff([])
         else:
             self.info('No more update to the database. Leaving.')
 
-    def _first_pbf_import(self):
+    def _first_pbf_import(self, args):
         """Run the first PBF import into the database."""
         command = ['imposm', 'import', '-diff', '-deployproduction']
         command += ['-overwritecache', '-cachedir', self.default['CACHE']]
@@ -295,7 +270,8 @@ class Importer(object):
         command += ['-read', self.osm_file]
         command += ['-write', '-connection', self.postgis_uri]
         self.info('The database is empty. Let\'s import the PBF : %s' % self.osm_file)
-        self.info(' '.join(command))
+
+        self.info(command.extend(args))
         if not call(command) == 0:
             msg = 'An error occured in imposm with the original file.'
             self.error(msg)
@@ -309,14 +285,10 @@ class Importer(object):
         if self.post_import_file:
             self.import_custom_sql()
 
-        if self.clip_shape_file:
-            self._import_clip_function()
-            self.perform_clip_in_db()
-
         if self.qgis_style:
             self.import_qgis_styles()
 
-    def _import_diff(self):
+    def _import_diff(self, args):
         # Finally launch the listening process.
         while True:
             import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
@@ -331,10 +303,11 @@ class Importer(object):
                     command += ['-srid', self.default['SRID']]
                     command += ['-diffdir', self.default['SETTINGS']]
                     command += ['-mapping', self.mapping_file]
+                    command += ['-limitto', self.clip_json_file]
                     command += ['-connection', self.postgis_uri]
                     command += [join(self.default['IMPORT_QUEUE'], diff)]
 
-                    self.info(' '.join(command))
+                    self.info(command.extend(args))
 
                     if call(command) == 0:
                         move(
@@ -344,11 +317,6 @@ class Importer(object):
                         # Update the timestamp in the file.
                         database_timestamp = diff.split('.')[0].split('->-')[1]
                         self.update_timestamp(database_timestamp)
-
-                        if self.clip_shape_file:
-                            self.perform_clip_in_db()
-
-                        self.info('Import diff successful : %s' % diff)
                     else:
                         msg = 'An error occured in imposm with a diff.'
                         self.error(msg)

readme.md

@@ -2,7 +2,8 @@
 
 A docker compose project to setup an OSM PostGIS database with automatic
 updates from OSM periodically.
-The only file you need is a PBF file and run the docker compose project.
+The only files you need is a PBF file, geojson (if you intent to restrict data download to
+a smaller extent than the one specified by the PBF) and run the docker compose project.
 
 ## General architecture
 
@@ -83,14 +84,19 @@ you don't set a clipping area, you will end with data from all over the world.
 
 ### Clipping
 
-You can put a shapefile in the clip folder. This shapefile will be 
+During the initial import or post update imposm uses the flag `-limito` which allows 
-used for clipping every features after the import.
+you to define a smaller area that you can work with.  
-This file has to be named 'clip.shp' and in the CRS you are using in the database (4326 by default).
+This is always desirable to limit the features being imported into the database rather 
-When the database container is running, import the shapefile in the database using the command : 
+than clipping them.
 
-`make import_clip`.
+**NB:** Ensure you add a geojson covering the area you intent to clip into the settings folder.
+The geojson can be the same extent of the administrative area of your country or it can be a 
+smaller extent. The CRS of the geojson should always be EPSG:4326.
 
-You can remove the clip file : `make remove_clip`.
+
+**NB:** It is encouraged to simplify the geometry for the `clip.geojson` as
+a simplified geometry is easier to process during the import. 
+Rather use the minimum bounding box for the area you intent to clip your dataset with.
 
 ### QGIS Styles
 
@@ -239,16 +245,24 @@ With -e, you can add some settings to PostGIS:
 ```bash
  - ALLOW_IP_RANGE= 0.0.0.0/0
 ```
+More environment variables for Kartoza/postgis image can be found from https://github.com/kartoza/docker-postgis#environment-variables
 
-# QGIS Server
+# QGIS Server and Martin Vector tiles
 
-You can run a QGIS Server front end to the OSM mirroir by using the provided
+You can run a QGIS Server front end or martin vector tiles to the OSM mirror by using the provided
 docker-compose-web.yml file. For example:
 
 ```bash
 docker-compose -f docker-compose.yml -f docker-compose-web.yml qgisserver  up
 ```
 
+or
+```bash
+docker-compose -f docker-compose.yml -f docker-compose-web.yml martin  up
+```
+For more information about martin configuration and usage can be found from https://github.com/urbica/martin
+
+
 # Credits
 
 This application was designed and implemented by:

settings/clip/clip.sql

@@ -1,38 +0,0 @@
-CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
-$BODY$
-DECLARE osm_tables CURSOR FOR
-    SELECT table_name
-    FROM information_schema.tables
-    WHERE table_schema='public'
-    AND table_type='BASE TABLE'
-    AND table_name LIKE 'osm_%';
-BEGIN
-    FOR osm_table IN osm_tables LOOP
-        EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
-            SELECT DISTINCT osm_id
-            FROM ' || quote_ident(osm_table.table_name) || '
-            LEFT JOIN clip ON ST_Intersects(geometry, geom) where clip.ogc_fid is NULL)
-        ;';
-    END LOOP;
-END;
-$BODY$
-LANGUAGE plpgsql;
-
--- Function to validate geometry of all tables.
--- To run it after creating the function simply run SELECT validate_geom();
-CREATE OR REPLACE FUNCTION validate_geom() RETURNS void AS
-$BODY$
-DECLARE osm_tables CURSOR FOR
-    SELECT table_name
-    FROM information_schema.tables
-    WHERE table_schema='public'
-    AND table_type='BASE TABLE'
-    AND table_name LIKE 'osm_%';
-BEGIN
-    FOR osm_table IN osm_tables LOOP
-        EXECUTE 'UPDATE ' || quote_ident(osm_table.table_name) || ' SET
-              geometry = ST_MakeValid(geometry) where ST_IsValidReason(geometry) = ''F'' ;';
-    END LOOP;
-END;
-$BODY$
-LANGUAGE plpgsql;