From c413e18fb75c127132e1f8afb59c590aa8058491 Mon Sep 17 00:00:00 2001 From: Etienne Trimaille Date: Fri, 18 Dec 2015 15:05:30 +0100 Subject: [PATCH] cleaning and improve log --- docker-compose.yml | 12 +-- docker-imposm3/importer.py | 153 +++++++++++++++++++---------------- docker-osmupdate/download.py | 17 ++-- 3 files changed, 99 insertions(+), 83 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 573a9d2..fadc598 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,9 @@ storage: - /home/import_done - /home/import_queue - /home/cache + #- ./import_done:/home/import_done + #- ./import_queue:/home/import_queue + #- ./cache:/home/cache db: image: kartoza/postgis:9.4-2.1 @@ -34,8 +37,6 @@ imposm: - SETTINGS = settings # folder for caching - CACHE = cache - # folder the OSM file - - BASE_PBF = base_pbf # folder for diff which has been imported - IMPORT_DONE = import_done # folder for diff which hasn't been imported yet @@ -53,7 +54,6 @@ imposm: osmupdate: - #image: etrimaille/osmupdate build: docker-osmupdate volumes_from: - storage @@ -70,7 +70,9 @@ osmupdate: - COMPRESSION_LEVEL = 1 # change the URL to use a custom URL to fetch regional file updates. - BASE_URL = http://planet.openstreetmap.org/replication/ + # folder for diff which hasn't been imported yet - IMPORT_QUEUE = import_queue + # folder for diff which has been imported - IMPORT_DONE = import_done - - OSM_PBF = osm_pbf - - TIME = 120, seconds between each execution of the script + # seconds between 2 executions of the script + - TIME = 120 diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index c20c3bf..0513bd5 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -28,9 +28,6 @@ from subprocess import call from time import sleep from sys import stderr -# In docker-compose, we should wait for the DB is ready. -sleep(45) - # All these default values can be overwritten by env vars default = { 'TIME': 120, @@ -60,6 +57,57 @@ if default['SRID'] not in ['4326', '3857']: print >> stderr, 'SRID not supported : %s' % default['srid'] exit() +# Check folders. +folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE'] +for folder in folders: + if not isabs(default[folder]): + # Get the absolute path. + default[folder] = abspath(default[folder]) + + # Test the folder + if not exists(default[folder]): + print >> stderr, 'The folder %s does not exist.' % default[folder] + exit() + +# Test files +state_file = None +osm_file = None +mapping_file = None +post_import_file = None +for f in listdir(default['SETTINGS']): + if f == 'last.state.txt': + state_file = join(default['SETTINGS'], f) + + if f.endswith('.pbf'): + osm_file = join(default['SETTINGS'], f) + + if f.endswith('.json'): + mapping_file = join(default['SETTINGS'], f) + + if f.endswith('.sql'): + post_import_file = join(default['SETTINGS'], f) + +if not osm_file: + print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS'] + exit() + +if not state_file: + print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS'] + exit() + +if not mapping_file: + print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS'] + exit() + +if not post_import_file: + print 'No *.sql detected in %s' % default['SETTINGS'] +else: + print '%s detected for post import.' % post_import_file + +# In docker-compose, we should wait for the DB is ready. +print 'The checkup is OK. The container will continue soon, after the database.' +sleep(45) + # Check postgis. try: connection = connect( @@ -79,52 +127,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % ( default['HOST'], default['DATABASE']) -# Check folders. -folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE'] -for folder in folders: - if not isabs(default[folder]): - # Get the absolute path. - default[folder] = abspath(default[folder]) - # Test the folder - if not exists(default[folder]): - print >> stderr, 'The folder %s does not exist.' % default[folder] - exit() - -# Test files -state_file = None -osm_file = None -mapping_file = None -post_import_file = None -for f in listdir(default['SETTINGS']): - if f.endswith('.state.txt'): - state_file = join(default['SETTINGS'], f) - - if f.endswith('.pbf'): - osm_file = join(default['SETTINGS'], f) - - if f.endswith('.json'): - mapping_file = join(default['SETTINGS'], f) - - if f.endswith('.sql'): - post_import_file = join(default['SETTINGS'], f) - -if not osm_file: - print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS'] - exit() - -if not state_file: - print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS'] - exit() - -if not mapping_file: - print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS'] - exit() - -if not post_import_file: - print 'No *.sql detected in %s' % default['SETTINGS'] -else: - print '%s detected for post import.' % post_import_file # Check if there is a table starting with 'osm_' sql = 'select count(*) ' \ @@ -134,25 +137,31 @@ sql = 'select count(*) ' \ cursor.execute(sql) osm_tables = cursor.fetchone()[0] if osm_tables < 1: - # It means that the DB is empty. Let's import the file. - commands = ['imposm3', 'import', '-diff', '-deployproduction'] - commands += ['-overwritecache', '-cachedir', default['CACHE']] - commands += ['-srid', default['SRID']] - commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] - commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']] - commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] - commands += ['-diffdir', default['SETTINGS']] - commands += ['-mapping', mapping_file] - commands += ['-read', osm_file] - commands += ['-write', '-connection', postgis_uri] + # It means that the DB is empty. Let's import the PBF file. + command = ['imposm3', 'import', '-diff', '-deployproduction'] + command += ['-overwritecache', '-cachedir', default['CACHE']] + command += ['-srid', default['SRID']] + command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] + command += ['-dbschema-import', default['DBSCHEMA_IMPORT']] + command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] + command += ['-diffdir', default['SETTINGS']] + command += ['-mapping', mapping_file] + command += ['-read', osm_file] + command += ['-write', '-connection', postgis_uri] - if not call(commands) == 0: + print 'The database is empty. Let\'s import the PBF : %s' % osm_file + print ' '.join(command) + if not call(command) == 0: print >> stderr, 'An error occured in imposm with the original file.' exit() + else: + print 'Import PBF successful : %s' % osm_file if post_import_file: for sql in open(post_import_file): cursor.execute(sql) +else: + print 'The database is not empty. Let\'s import only diff files.' # Finally launch the listening process. while True: @@ -160,21 +169,23 @@ while True: if len(import_queue) > 0: for diff in import_queue: print 'Importing diff %s' % diff - commands = ['imposm3', 'diff'] - commands += ['-cachedir', default['CACHE']] - commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] - commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']] - commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] - commands += ['-srid', default['SRID']] - commands += ['-diffdir', default['SETTINGS']] - commands += ['-mapping', mapping_file] - commands += ['-connection', postgis_uri] - commands += [join(default['IMPORT_QUEUE'], diff)] + command = ['imposm3', 'diff'] + command += ['-cachedir', default['CACHE']] + command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] + command += ['-dbschema-import', default['DBSCHEMA_IMPORT']] + command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] + command += ['-srid', default['SRID']] + command += ['-diffdir', default['SETTINGS']] + command += ['-mapping', mapping_file] + command += ['-connection', postgis_uri] + command += [join(default['IMPORT_QUEUE'], diff)] - if call(commands) == 0: + print ' '.join(command) + if call(command) == 0: move( join(default['IMPORT_QUEUE'], diff), join(default['IMPORT_DONE'], diff)) + print 'Import diff successful : %s' % diff else: print >> stderr, 'An error occured in imposm with a diff.' exit() diff --git a/docker-osmupdate/download.py b/docker-osmupdate/download.py index bc3b99a..31dc5bd 100644 --- a/docker-osmupdate/download.py +++ b/docker-osmupdate/download.py @@ -19,7 +19,7 @@ ***************************************************************************/ """ -from os.path import exists, join, isabs, abspath, isfile +from os.path import exists, join, isabs, abspath from os import listdir, environ from sys import exit from subprocess import call @@ -28,6 +28,7 @@ from time import sleep from sys import stderr # In docker-compose, we should wait for the DB is ready. +print 'The container will start soon, after the database.' sleep(45) # Default values which can be overwritten. @@ -64,7 +65,7 @@ state_file = None osm_file = None poly_file = None for f in listdir(default['SETTINGS']): - if f.endswith('.state.txt'): + if f == 'last.state.txt': state_file = join(default['SETTINGS'], f) if f.endswith('.pbf'): @@ -81,7 +82,7 @@ for f in listdir(default['SETTINGS']): """ if not state_file: - print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS'] + print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS'] exit() if not osm_file: @@ -93,22 +94,22 @@ if not poly_file: else: print '%s detected for clipping.' % poly_file +# Finally launch the listening process. while True: # Check if diff to be imported is empty. If not, take the latest diff. diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE'])) if len(diff_to_be_imported): - print "Timestamp from the lastest not imported diff." timestamp = diff_to_be_imported[-1].split('.')[0] + print "Timestamp from the latest not imported diff : %s" % timestamp else: # Check if imported diff is empty. If not, take the latest diff. imported_diff = sorted(listdir(default['IMPORT_DONE'])) if len(imported_diff): - print "Timestamp from the lastest imported diff." + print "Timestamp from the latest imported diff : %s" % timestamp timestamp = imported_diff[-1].split('.')[0] else: # Take the timestamp from original file. - print "Timestamp from the original state file." state_file_settings = {} with open(state_file) as a_file: for line in a_file: @@ -117,6 +118,7 @@ while True: state_file_settings[name] = value timestamp = state_file_settings['timestamp'].strip() + print "Timestamp from the original state file : %s" % timestamp # Removing some \ in the timestamp. timestamp = timestamp.replace('\\', '') @@ -142,8 +144,9 @@ while True: command.append(timestamp) command.append(file_path) + print ' '.join(command) if call(command) != 0: - print >> stderr, 'An error occured in osmupdate.' + print >> stderr, 'An error occured in osmupdate. Let\'s try again.' # Sleep less. print 'Sleeping for 2 seconds.' sleep(2.0)