refactoring osmupdate

pull/28/head
Etienne Trimaille 2015-12-29 14:05:57 +01:00
rodzic 4c1899854f
commit c3692553c2
1 zmienionych plików z 113 dodań i 91 usunięć

Wyświetl plik

@ -27,8 +27,12 @@ from datetime import datetime
from time import sleep
from sys import stderr
class Downloader(object):
def __init__(self):
# Default values which can be overwritten.
default = {
self.default = {
'MAX_DAYS': '100',
'DIFF': 'sporadic',
'MAX_MERGE': '7',
@ -39,101 +43,119 @@ default = {
'SETTINGS': 'settings',
'TIME': 120,
}
self.osm_file = None
@staticmethod
def info(message):
print message
@staticmethod
def error(message):
print >> stderr, message
exit()
def overwrite_environment(self):
"""Overwrite default values from the environment."""
for key in environ.keys():
if key in default.keys():
default[key] = environ[key]
if key in self.default.keys():
self.default[key] = environ[key]
def check_settings(self):
"""Perform various checking."""
# Folders
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS']
for folder in folders:
if not isabs(default[folder]):
if not isabs(self.default[folder]):
# Get the absolute path.
default[folder] = abspath(default[folder])
self.default[folder] = abspath(self.default[folder])
# Test the folder
if not exists(default[folder]):
print >> stderr, 'The folder %s does not exist.' % default[folder]
exit()
if not exists(self.default[folder]):
msg = 'The folder %s does not exist.' % self.default[folder]
self.error(msg)
# Test files
osm_file = None
for f in listdir(default['SETTINGS']):
for f in listdir(self.default['SETTINGS']):
if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f)
self.osm_file = join(self.default['SETTINGS'], f)
"""
# Todo : need fix custom URL and sporadic diff : daily, hourly and minutely
if f == 'custom_url_diff.txt':
with open(join(default['SETTINGS'], f), 'r') as content_file:
default['BASE_URL'] = content_file.read()
"""
if not self.osm_file:
msg = 'OSM file *.osm.pbf is missing in %s' % self.default['SETTINGS']
self.error(msg)
if not osm_file:
print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS']
exit()
# In docker-compose, we should wait for the DB is ready.
print 'The checkup is OK. The container will continue soon, after the database.'
self.info('The checkup is OK. The container will continue soon, after the database.')
sleep(45)
# Finally launch the listening process.
while True:
def _check_latest_timestamp(self):
"""Fetch the latest timestamp."""
# Check if diff to be imported is empty. If not, take the latest diff.
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
diff_to_be_imported = sorted(listdir(self.default['IMPORT_QUEUE']))
if len(diff_to_be_imported):
file_name = diff_to_be_imported[-1].split('.')[0]
timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest not imported diff : %s' % timestamp
self.info('Timestamp from the latest not imported diff : %s' % timestamp)
else:
# Check if imported diff is empty. If not, take the latest diff.
imported_diff = sorted(listdir(default['IMPORT_DONE']))
imported_diff = sorted(listdir(self.default['IMPORT_DONE']))
if len(imported_diff):
file_name = imported_diff[-1].split('.')[0]
timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest imported diff : %s' % timestamp
self.info('Timestamp from the latest imported diff : %s' % timestamp)
else:
# Take the timestamp from original file.
command = ['osmconvert', osm_file, '--out-timestamp']
processus = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
command = ['osmconvert', self.osm_file, '--out-timestamp']
processus = Popen(
command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
timestamp, err = processus.communicate()
# Remove new line
timestamp = timestamp.strip()
print 'Timestamp from the original state file : %s' % timestamp
self.info('Timestamp from the original state file : %s' % timestamp)
# Removing some \ in the timestamp.
timestamp = timestamp.replace('\\', '')
return timestamp
def download(self):
"""Infinite loop to download diff files on a regular interval."""
while True:
timestamp = self._check_latest_timestamp()
# Save time
current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
print 'Old time : %s' % timestamp
print 'Current time : %s' % current_time
self.info('Old time : %s' % timestamp)
self.info('Current time : %s' % current_time)
# Destination
file_name = '%s->-%s.osc.gz' % (timestamp, current_time)
file_path = join(default['IMPORT_QUEUE'], file_name)
file_path = join(self.default['IMPORT_QUEUE'], file_name)
# Command
command = ['osmupdate', '-v']
command += ['--max-days=' + default['MAX_DAYS']]
command += [default['DIFF']]
command += ['--max-merge=' + default['MAX_MERGE']]
command += ['--compression-level=' + default['COMPRESSION_LEVEL']]
command += ['--base-url=' + default['BASE_URL']]
command += ['--max-days=' + self.default['MAX_DAYS']]
command += [self.default['DIFF']]
command += ['--max-merge=' + self.default['MAX_MERGE']]
command += ['--compression-level=' + self.default['COMPRESSION_LEVEL']]
command += ['--base-url=' + self.default['BASE_URL']]
command.append(timestamp)
command.append(file_path)
print ' '.join(command)
self.info(' '.join(command))
if call(command) != 0:
print >> stderr, 'An error occured in osmupdate. Let\'s try again.'
self.info('An error occured in osmupdate. Let\'s try again.')
# Sleep less.
print 'Sleeping for 2 seconds.'
self.info('Sleeping for 2 seconds.')
sleep(2.0)
else:
# Everything was fine, let's sleeping.
print 'Sleeping for %s seconds.' % default['TIME']
sleep(float(default['TIME']))
self.info('Sleeping for %s seconds.' % self.default['TIME'])
sleep(float(self.default['TIME']))
if __name__ == '__main__':
downloader = Downloader()
downloader.overwrite_environment()
downloader.check_settings()
downloader.download()