Fix osm enrich, add env file

.env

@@ -0,0 +1,53 @@
+COMPOSE_PROJECT_NAME=kartozadockerosm
+POSTGRES_USER=docker
+POSTGRES_PASS=docker
+POSTGRES_DBNAME=gis
+DB_PORT=35432
+PGDB_PORT=6500
+POSTGRES_VERSION=13-3.1
+MARTIN_PORT=3000
+WATCH_MODE=true
+DATABASE_URL=postgres://docker:docker@db/gis
+# Uncomment to expose the postgis database on the network
+ALLOW_IP_RANGE= 0.0.0.0/0
+POSTGRES_PORT=5432
+POSTGRES_HOST=db
+# seconds between 2 executions of the script
+# if 0, then no update will be done, only the first initial import from the PBF
+TIME=120
+# folder for settings (with *.json and *.sql)
+SETTINGS=settings
+# folder for caching
+CACHE=cache
+# folder for diff which has been imported
+IMPORT_DONE=import_done
+# folder for diff which hasn't been imported yet
+IMPORT_QUEUE=import_queue
+# it can be 3857
+SRID=4326
+# see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
+OPTIMIZE=false
+# see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+DBSCHEMA_PRODUCTION=public
+# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+DBSCHEMA_IMPORT=import
+# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+DBSCHEMA_BACKUP=backup
+# Install some styles if you are using the default mapping. It can be 'yes' or 'no'
+QGIS_STYLE=yes
+# Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
+CLIP=no
+# These are all currently the defaults but listed here for your
+# convenience if you want to change them
+# the maximum time range to assemble a cumulated changefile.
+MAX_DAYS=100
+# osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+DIFF=sporadic
+# argument to determine the maximum number of parallely processed changefiles.
+MAX_MERGE=7
+# define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
+COMPRESSION_LEVEL=1
+# change the URL to use a custom URL to fetch regional file updates.
+BASE_URL=http://planet.openstreetmap.org/replication/
+PGADMIN_DEFAULT_EMAIL=docker@gmail.com
+PGADMIN_DEFAULT_PASSWORD=docker

docker-compose-web.yml

@@ -1,42 +1,112 @@
-# Usage:
+version: '3.9'
 
-# docker-compose -f docker-compose.yml -f docker-compose-web.yml
+volumes:
-
+  osm-postgis-data:
-version: '2.1'
+  import_done:
+  import_queue:
+  cache:
+  pgadmin_data:
 
 services:
-  qgisserver:
+  db:
-    image: camptocamp/qgis-server:3.6
+    image: kartoza/postgis:${POSTGRES_VERSION}
-    hostname: dockerosm_qgisserver
+    hostname: db
-    container_name: dockerosm_qgisserver
+    container_name: dockerosm_db
     environment:
-      - QGIS_PROJECT_FILE=/project/project.qgs
+      - POSTGRES_USER=${POSTGRES_USER}
-      - GIS_SERVER_LOG_LEVEL=DEBUG
+      - POSTGRES_PASS=${POSTGRES_PASS}
-      - MAX_REQUESTS_PER_PROCESS=100
+      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
+      - ALLOW_IP_RANGE=${ALLOW_IP_RANGE}
     volumes:
-      - ./logs:/var/log/apache2
+      - osm-postgis-data:/var/lib/postgresql
-      - ./web:/project
-      - ./settings:/web/settings
-    depends_on:
-      db:
-        condition: service_healthy
-    links:
-      - db:db
     ports:
-      - 8198:80
+      - ${DB_PORT}:5432
-    restart: on-failure
+    healthcheck:
+      test: "exit 0"
+
+  imposm:
+    image: kartoza/docker-osm:imposm-latest
+    build: docker-imposm3
+    container_name: dockerosm_imposm
+    volumes:
+      - ./settings:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
+    depends_on:
+      - db
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASS=${POSTGRES_PASS}
+      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - TIME=${TIME}
+      - SETTINGS=${SETTINGS}
+      - CACHE=${CACHE}
+      - IMPORT_DONE=${IMPORT_DONE}
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
+      - SRID=${SRID}
+      - OPTIMIZE=${OPTIMIZE}
+      - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION}
+      - DBSCHEMA_IMPORT=${DBSCHEMA_IMPORT}
+      - DBSCHEMA_BACKUP=${DBSCHEMA_BACKUP}
+      - QGIS_STYLE=${QGIS_STYLE}
+      - CLIP=${CLIP}
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
+
+  osmupdate:
+    build: docker-osmupdate
+    image: kartoza/docker-osm:osmupdate-latest
+    container_name: dockerosm_osmupdate
+    volumes_from:
+      - imposm
+    depends_on:
+      - db
+    environment:
+      - MAX_DAYS=${MAX_DAYS}
+      - DIFF=${DIFF}
+      - MAX_MERGE=${MAX_MERGE}
+      - COMPRESSION_LEVEL=${COMPRESSION_LEVEL}
+      - BASE_URL=${BASE_URL}
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
+      - IMPORT_DONE=${IMPORT_DONE}
+      - TIME=${TIME}
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
+
+  pgadmin4:
+    image: dpage/pgadmin4:4.16
+    hostname: pgadmin4
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+    environment:
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
+      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD}
+    ports:
+      - ${PGDB_PORT}:80
+    restart: on-failure
+    depends_on:
+      - db
+
+  osmenrich:
+    build: docker-osmenrich
+    volumes_from:
+      - imposm
+    depends_on:
+      - db
+    environment:
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
+      - IMPORT_DONE=${IMPORT_DONE}
+      - TIME=${TIME}
 
-  # Server vector tiles from PostgreSQL DB
   martin:
     image: urbica/martin
-    hostname: dockerosm_martin
-    container_name: dockerosm_martin
     restart: on-failure
     ports:
-      - 3000:3000
+      - ${MARTIN_PORT}:3000
     environment:
-      - WATCH_MODE=true
+      - WATCH_MODE=${WATCH_MODE}
-      - DATABASE_URL=postgres://docker:docker@db/gis
+      - DATABASE_URL=${DATABASE_URL}
     depends_on:
       db:
         condition: service_healthy

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.9'
 
 volumes:
   osm-postgis-data:
@@ -9,21 +9,18 @@ volumes:
 
 services:
   db:
-    # About the postgresql version, it should match in the dockerfile of docker-imposm3
+    image: kartoza/postgis:${POSTGRES_VERSION}
-    image: kartoza/postgis:12.0
     hostname: db
     container_name: dockerosm_db
     environment:
-      - POSTGRES_USER=docker
+      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASS=docker
+      - POSTGRES_PASS=${POSTGRES_PASS}
-      - POSTGRES_DBNAME=gis
+      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
-      # Uncomment to expose the postgis database on the network
+      - ALLOW_IP_RANGE=${ALLOW_IP_RANGE}
-      # - ALLOW_IP_RANGE= 0.0.0.0/0
     volumes:
       - osm-postgis-data:/var/lib/postgresql
-    # Uncomment to use the postgis database from outside the docker network
+    ports:
-    # ports:
+      - ${DB_PORT}:5432
-    #  - "35432:5432"
     healthcheck:
       test: "exit 0"
 
@@ -33,80 +30,49 @@ services:
     build: docker-imposm3
     container_name: dockerosm_imposm
     volumes:
-      # These are sharable to other containers
       - ./settings:/home/settings
       - import_done:/home/import_done
       - import_queue:/home/import_queue
       - cache:/home/cache
     depends_on:
-      db:
+      - db
-        condition: service_healthy
     environment:
-      - POSTGRES_USER=docker
+      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASS=docker
+      - POSTGRES_PASS=${POSTGRES_PASS}
-      - POSTGRES_DBNAME=gis
+      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
-      - POSTGRES_PORT=5432
+      - POSTGRES_PORT=${POSTGRES_PORT}
-      - POSTGRES_HOST=db
+      - POSTGRES_HOST=${POSTGRES_HOST}
-      # seconds between 2 executions of the script
+      - TIME=${TIME}
-      # if 0, then no update will be done, only the first initial import from the PBF
+      - SETTINGS=${SETTINGS}
-      - TIME=120
+      - CACHE=${CACHE}
-      # folder for settings (with *.json and *.sql)
+      - IMPORT_DONE=${IMPORT_DONE}
-      - SETTINGS=settings
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      # folder for caching
+      - SRID=${SRID}
-      - CACHE=cache
+      - OPTIMIZE=${OPTIMIZE}
-      # folder for diff which has been imported
+      - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION}
-      - IMPORT_DONE=import_done
+      - DBSCHEMA_IMPORT=${DBSCHEMA_IMPORT}
-      # folder for diff which hasn't been imported yet
+      - DBSCHEMA_BACKUP=${DBSCHEMA_BACKUP}
-      - IMPORT_QUEUE=import_queue
+      - QGIS_STYLE=${QGIS_STYLE}
-      # it can be 3857
+      - CLIP=${CLIP}
-      - SRID=4326
-      # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
-      - OPTIMIZE=false
-      # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_PRODUCTION=public
-      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_IMPORT=import
-      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_BACKUP=backup
-      # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
-      - QGIS_STYLE=yes
-      # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
-      - CLIP=no
     command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
 
   osmupdate:
     build: docker-osmupdate
     image: kartoza/docker-osm:osmupdate-latest
     container_name: dockerosm_osmupdate
-    volumes:
+    volumes_from:
-      # These are sharable to other containers
+      - imposm
-      - ./settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
     depends_on:
-      db:
+      - db
-        condition: service_healthy
     environment:
-      # These are all currently the defaults but listed here for your
+      - MAX_DAYS=${MAX_DAYS}
-      # convenience if you want to change them
+      - DIFF=${DIFF}
-      # the maximum time range to assemble a cumulated changefile.
+      - MAX_MERGE=${MAX_MERGE}
-      - MAX_DAYS=100
+      - COMPRESSION_LEVEL=${COMPRESSION_LEVEL}
-      # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+      - BASE_URL=${BASE_URL}
-      - DIFF=sporadic
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      # argument to determine the maximum number of parallely processed changefiles.
+      - IMPORT_DONE=${IMPORT_DONE}
-      - MAX_MERGE=7
+      - TIME=${TIME}
-      # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
-      - COMPRESSION_LEVEL=1
-      # change the URL to use a custom URL to fetch regional file updates.
-      - BASE_URL=http://planet.openstreetmap.org/replication/
-      # folder for diff which hasn't been imported yet
-      - IMPORT_QUEUE=import_queue
-      # folder for diff which has been imported
-      - IMPORT_DONE=import_done
-      # seconds between 2 executions of the script
-      # if 0, then no update will be done, only the first initial import from the PBF
-      - TIME=120
     command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
 
   pgadmin4:
@@ -115,35 +81,22 @@ services:
     volumes:
       - pgadmin_data:/var/lib/pgadmin
     environment:
-      - PGADMIN_DEFAULT_EMAIL=docker@gmail.com
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
-      - PGADMIN_DEFAULT_PASSWORD=docker
+      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD}
     ports:
-      - 6500:80
+      - ${PGDB_PORT}:80
     restart: on-failure
     depends_on:
-      db:
+      - db
-        condition: service_healthy
 
   osmenrich:
     build: docker-osmenrich
     container_name: dockerosm_osmenrich
-    volumes:
+    volumes_from:
-      # These are sharable to other containers
+      - imposm
-      - ./settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
     depends_on:
-      db:
+      - db
-        condition: service_healthy
     environment:
-      # These are all currently the defaults but listed here for your
+      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      # convenience if you want to change them
+      - IMPORT_DONE=${IMPORT_DONE}
-      # folder for diff which hasn't been imported yet
+      - TIME=${TIME}
-      - IMPORT_QUEUE=import_queue
-      # folder for diff which has been imported
-      - IMPORT_DONE=import_done
-      # seconds between 2 executions of the script
-      # if 0, then no update will be done, only the first initial import from the PBF
-      - TIME=120
-    command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py"

docker-osmenrich/enrich.py

@@ -20,16 +20,17 @@
 """
 
 import gzip
-import xmltodict
-import yaml
-from xmltodict import OrderedDict
-from dateutil import parser
 from os import environ, listdir, mkdir
 from os.path import join, exists, getsize
 from sys import exit, stderr
-from urllib import request
-from psycopg2 import connect, OperationalError, ProgrammingError
 from time import sleep
+from urllib import request
+
+import xmltodict
+import yaml
+from dateutil import parser
+from psycopg2 import connect, OperationalError, ProgrammingError
+from xmltodict import OrderedDict
 
 
 class Enrich(object):
@@ -220,18 +221,22 @@ class Enrich(object):
             for table, table_data in self.mapping_database_schema.items():
                 new_columns_postgis = []
                 for enrich_key, enrich_type in self.enriched_column.items():
-                    try:
+                    check_column = ''' SELECT EXISTS (SELECT 1 FROM information_schema.columns 
-                        cursor.execute('select %s from %s' % (enrich_key, table))
+                                                                WHERE table_name='%s' and column_name='%s'); ''' % (
-                    except ProgrammingError as e:
+                        table, enrich_key)
-                        connection.rollback()
+                    cursor.execute(check_column)
+                    column_existence = cursor.fetchone()[0]
+
+                    if column_existence != 1:
                         if enrich_type == 'int':
-                            new_columns_postgis.append('ADD COLUMN %s INTEGER' % enrich_key)
+                            new_columns_postgis.append('ADD COLUMN IF NOT EXISTS %s NUMERIC' % enrich_key)
                         elif enrich_type == 'string':
-                            new_columns_postgis.append('ADD COLUMN %s VARCHAR' % enrich_key)
+                            new_columns_postgis.append('ADD COLUMN IF NOT EXISTS %s CHARACTER VARYING (255)' % enrich_key)
                         elif enrich_type == 'datetime':
-                            new_columns_postgis.append('ADD COLUMN %s timestamp' % enrich_key)
+                            new_columns_postgis.append('ADD COLUMN IF NOT EXISTS %s TIMESTAMPTZ' % enrich_key)
+
                 if len(new_columns_postgis) > 0:
-                    query = 'ALTER TABLE %s %s' % (table, ','.join(new_columns_postgis))
+                    query = 'ALTER TABLE public."%s" %s;' % (table, ','.join(new_columns_postgis))
                     cursor.execute(query)
                     connection.commit()
             connection.close()
@@ -415,8 +420,8 @@ class Enrich(object):
         row_batch = {}
         osm_ids = []
         try:
-            cursor.execute(
+            check_sql = ''' select * from "%s" WHERE "changeset_timestamp" IS NULL AND "osm_id" IS NOT NULL ORDER BY "osm_id" ''' % table_name
-                'select * from %s WHERE changeset_timestamp IS NULL AND osm_id IS NOT NULL ORDER BY osm_id' % table_name)
+            cursor.execute(check_sql)
             row = True
             while row:
                 # do something with row
@@ -479,8 +484,9 @@ class Enrich(object):
                 connection = self.create_connection()
                 cursor = connection.cursor()
                 try:
-                    cursor.execute('select * from %s WHERE %s=%s' % (
+                    validate_sql = ''' select * from "%s" WHERE "%s"=%s  ''' % (
-                        table, table_data['osm_id_columnn'], osm_id))
+                        table, table_data['osm_id_columnn'], osm_id)
+                    cursor.execute(validate_sql)
                     row = cursor.fetchone()
                     if row:
                         row = dict(zip(table_data['columns'], row))
@@ -550,15 +556,26 @@ class Enrich(object):
                 except IOError:
                     self.info('cache file can\'t be created')
 
+    def locate_table(self, name):
+        """Check for tables in the DB table exists in the DB"""
+        connection = self.create_connection()
+        cursor = connection.cursor()
+        sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like  'TEMP_TABLE'); """
+        cursor.execute(sql.replace('TEMP_TABLE', '%s' % name))
+        # noinspection PyUnboundLocalVariable
+        return cursor.fetchone()[0]
+
     def run(self):
         """First checker."""
         while True:
             self.info('Run enrich process')
-            if self.check_database():
+            osm_tables = self.locate_table('osm_%')
-                self.enrich_empty_changeset()
+            if osm_tables != 1:
-                self.enrich_database_from_diff_file()
+                self.info('Imposm is still running, wait a while and try again')
             else:
-                self.info('Database is not ready')
+                if self.check_database():
+                    self.enrich_empty_changeset()
+                    self.enrich_database_from_diff_file()
 
             # sleep looping
             self.info('sleeping for %s' % self.default['TIME'])