cleaning and improve log

docker-compose.yml

@@ -7,6 +7,9 @@ storage:
     - /home/import_done
     - /home/import_queue
     - /home/cache
+    #- ./import_done:/home/import_done
+    #- ./import_queue:/home/import_queue
+    #- ./cache:/home/cache
 
 db:
   image: kartoza/postgis:9.4-2.1
@@ -34,8 +37,6 @@ imposm:
     - SETTINGS = settings
     # folder for caching
     - CACHE = cache
-    # folder the OSM file
-    - BASE_PBF = base_pbf
     # folder for diff which has been imported
     - IMPORT_DONE = import_done
     # folder for diff which hasn't been imported yet
@@ -53,7 +54,6 @@ imposm:
 
 
 osmupdate:
-  #image: etrimaille/osmupdate
   build: docker-osmupdate
   volumes_from:
     - storage
@@ -70,7 +70,9 @@ osmupdate:
     - COMPRESSION_LEVEL = 1
     # change the URL to use a custom URL to fetch regional file updates.
     - BASE_URL = http://planet.openstreetmap.org/replication/
+    # folder for diff which hasn't been imported yet
     - IMPORT_QUEUE = import_queue
+    # folder for diff which has been imported
     - IMPORT_DONE = import_done
-    - OSM_PBF = osm_pbf
+    # seconds between 2 executions of the script
-    - TIME = 120, seconds between each execution of the script
+    - TIME = 120

docker-imposm3/importer.py

@@ -28,9 +28,6 @@ from subprocess import call
 from time import sleep
 from sys import stderr
 
-# In docker-compose, we should wait for the DB is ready.
-sleep(45)
-
 # All these default values can be overwritten by env vars
 default = {
     'TIME': 120,
@@ -60,6 +57,57 @@ if default['SRID'] not in ['4326', '3857']:
     print >> stderr, 'SRID not supported : %s' % default['srid']
     exit()
 
+# Check folders.
+folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
+for folder in folders:
+    if not isabs(default[folder]):
+        # Get the absolute path.
+        default[folder] = abspath(default[folder])
+
+    # Test the folder
+    if not exists(default[folder]):
+        print >> stderr, 'The folder %s does not exist.' % default[folder]
+        exit()
+
+# Test files
+state_file = None
+osm_file = None
+mapping_file = None
+post_import_file = None
+for f in listdir(default['SETTINGS']):
+    if f == 'last.state.txt':
+        state_file = join(default['SETTINGS'], f)
+
+    if f.endswith('.pbf'):
+        osm_file = join(default['SETTINGS'], f)
+
+    if f.endswith('.json'):
+        mapping_file = join(default['SETTINGS'], f)
+
+    if f.endswith('.sql'):
+        post_import_file = join(default['SETTINGS'], f)
+
+if not osm_file:
+    print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
+    exit()
+
+if not state_file:
+    print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
+    exit()
+
+if not mapping_file:
+    print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
+    exit()
+
+if not post_import_file:
+    print 'No *.sql detected in %s' % default['SETTINGS']
+else:
+    print '%s detected for post import.' % post_import_file
+
+# In docker-compose, we should wait for the DB is ready.
+print 'The checkup is OK. The container will continue soon, after the database.'
+sleep(45)
+
 # Check postgis.
 try:
     connection = connect(
@@ -79,52 +127,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % (
     default['HOST'],
     default['DATABASE'])
 
-# Check folders.
-folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
-for folder in folders:
-    if not isabs(default[folder]):
-        # Get the absolute path.
-        default[folder] = abspath(default[folder])
 
-    # Test the folder
-    if not exists(default[folder]):
-        print >> stderr, 'The folder %s does not exist.' % default[folder]
-        exit()
-
-# Test files
-state_file = None
-osm_file = None
-mapping_file = None
-post_import_file = None
-for f in listdir(default['SETTINGS']):
-    if f.endswith('.state.txt'):
-        state_file = join(default['SETTINGS'], f)
-
-    if f.endswith('.pbf'):
-        osm_file = join(default['SETTINGS'], f)
-
-    if f.endswith('.json'):
-        mapping_file = join(default['SETTINGS'], f)
-
-    if f.endswith('.sql'):
-        post_import_file = join(default['SETTINGS'], f)
-
-if not osm_file:
-    print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
-    exit()
-
-if not state_file:
-    print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
-    exit()
-
-if not mapping_file:
-    print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
-    exit()
-
-if not post_import_file:
-    print 'No *.sql detected in %s' % default['SETTINGS']
-else:
-    print '%s detected for post import.' % post_import_file
 
 # Check if there is a table starting with 'osm_'
 sql = 'select count(*) ' \
@@ -134,25 +137,31 @@ sql = 'select count(*) ' \
 cursor.execute(sql)
 osm_tables = cursor.fetchone()[0]
 if osm_tables < 1:
-    # It means that the DB is empty. Let's import the file.
+    # It means that the DB is empty. Let's import the PBF file.
-    commands = ['imposm3', 'import', '-diff', '-deployproduction']
+    command = ['imposm3', 'import', '-diff', '-deployproduction']
-    commands += ['-overwritecache', '-cachedir', default['CACHE']]
+    command += ['-overwritecache', '-cachedir', default['CACHE']]
-    commands += ['-srid', default['SRID']]
+    command += ['-srid', default['SRID']]
-    commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
+    command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
-    commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
+    command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
-    commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
+    command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
-    commands += ['-diffdir', default['SETTINGS']]
+    command += ['-diffdir', default['SETTINGS']]
-    commands += ['-mapping', mapping_file]
+    command += ['-mapping', mapping_file]
-    commands += ['-read', osm_file]
+    command += ['-read', osm_file]
-    commands += ['-write', '-connection', postgis_uri]
+    command += ['-write', '-connection', postgis_uri]
 
-    if not call(commands) == 0:
+    print 'The database is empty. Let\'s import the PBF : %s' % osm_file
+    print ' '.join(command)
+    if not call(command) == 0:
         print >> stderr, 'An error occured in imposm with the original file.'
         exit()
+    else:
+        print 'Import PBF successful : %s' % osm_file
 
     if post_import_file:
         for sql in open(post_import_file):
             cursor.execute(sql)
+else:
+    print 'The database is not empty. Let\'s import only diff files.'
 
 # Finally launch the listening process.
 while True:
@@ -160,21 +169,23 @@ while True:
     if len(import_queue) > 0:
         for diff in import_queue:
             print 'Importing diff %s' % diff
-            commands = ['imposm3', 'diff']
+            command = ['imposm3', 'diff']
-            commands += ['-cachedir', default['CACHE']]
+            command += ['-cachedir', default['CACHE']]
-            commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
+            command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
-            commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
+            command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
-            commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
+            command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
-            commands += ['-srid', default['SRID']]
+            command += ['-srid', default['SRID']]
-            commands += ['-diffdir', default['SETTINGS']]
+            command += ['-diffdir', default['SETTINGS']]
-            commands += ['-mapping', mapping_file]
+            command += ['-mapping', mapping_file]
-            commands += ['-connection', postgis_uri]
+            command += ['-connection', postgis_uri]
-            commands += [join(default['IMPORT_QUEUE'], diff)]
+            command += [join(default['IMPORT_QUEUE'], diff)]
 
-            if call(commands) == 0:
+            print ' '.join(command)
+            if call(command) == 0:
                 move(
                     join(default['IMPORT_QUEUE'], diff),
                     join(default['IMPORT_DONE'], diff))
+                print 'Import diff successful : %s' % diff
             else:
                 print >> stderr, 'An error occured in imposm with a diff.'
                 exit()

docker-osmupdate/download.py

@@ -19,7 +19,7 @@
  ***************************************************************************/
 """
 
-from os.path import exists, join, isabs, abspath, isfile
+from os.path import exists, join, isabs, abspath
 from os import listdir, environ
 from sys import exit
 from subprocess import call
@@ -28,6 +28,7 @@ from time import sleep
 from sys import stderr
 
 # In docker-compose, we should wait for the DB is ready.
+print 'The container will start soon, after the database.'
 sleep(45)
 
 # Default values which can be overwritten.
@@ -64,7 +65,7 @@ state_file = None
 osm_file = None
 poly_file = None
 for f in listdir(default['SETTINGS']):
-    if f.endswith('.state.txt'):
+    if f == 'last.state.txt':
         state_file = join(default['SETTINGS'], f)
 
     if f.endswith('.pbf'):
@@ -81,7 +82,7 @@ for f in listdir(default['SETTINGS']):
     """
 
 if not state_file:
-    print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
+    print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
     exit()
 
 if not osm_file:
@@ -93,22 +94,22 @@ if not poly_file:
 else:
     print '%s detected for clipping.' % poly_file
 
+# Finally launch the listening process.
 while True:
     # Check if diff to be imported is empty. If not, take the latest diff.
     diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
     if len(diff_to_be_imported):
-        print "Timestamp from the lastest not imported diff."
         timestamp = diff_to_be_imported[-1].split('.')[0]
+        print "Timestamp from the latest not imported diff : %s" % timestamp
     else:
         # Check if imported diff is empty. If not, take the latest diff.
         imported_diff = sorted(listdir(default['IMPORT_DONE']))
         if len(imported_diff):
-            print "Timestamp from the lastest imported diff."
+            print "Timestamp from the latest imported diff : %s" % timestamp
             timestamp = imported_diff[-1].split('.')[0]
 
         else:
             # Take the timestamp from original file.
-            print "Timestamp from the original state file."
             state_file_settings = {}
             with open(state_file) as a_file:
                 for line in a_file:
@@ -117,6 +118,7 @@ while True:
                         state_file_settings[name] = value
 
             timestamp = state_file_settings['timestamp'].strip()
+            print "Timestamp from the original state file : %s" % timestamp
 
     # Removing some \ in the timestamp.
     timestamp = timestamp.replace('\\', '')
@@ -142,8 +144,9 @@ while True:
     command.append(timestamp)
     command.append(file_path)
 
+    print ' '.join(command)
     if call(command) != 0:
-        print >> stderr, 'An error occured in osmupdate.'
+        print >> stderr, 'An error occured in osmupdate. Let\'s try again.'
         # Sleep less.
         print 'Sleeping for 2 seconds.'
         sleep(2.0)