mirror
/
docker-osm
kopia lustrzana https://github.com/kartoza/docker-osm
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

refactoring osmupdate

pull/28/head
Etienne Trimaille 2015-12-29 14:05:57 +01:00
rodzic 4c1899854f
commit c3692553c2
1 zmienionych plików z 113 dodań i 91 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

128
docker-osmupdate/download.py
Wyświetl plik

@ -27,8 +27,12 @@ from datetime import datetime
from time import sleep from time import sleep
from sys import stderr from sys import stderr
# Default values which can be overwritten.
default = { class Downloader(object):
def __init__(self):
# Default values which can be overwritten.
self.default = {
'MAX_DAYS': '100', 'MAX_DAYS': '100',
'DIFF': 'sporadic', 'DIFF': 'sporadic',
'MAX_MERGE': '7', 'MAX_MERGE': '7',
@ -38,102 +42,120 @@ default = {
'IMPORT_DONE': 'import_done', 'IMPORT_DONE': 'import_done',
'SETTINGS': 'settings', 'SETTINGS': 'settings',
'TIME': 120, 'TIME': 120,
} }
self.osm_file = None
for key in environ.keys(): @staticmethod
if key in default.keys(): def info(message):
default[key] = environ[key] print message
# Folders @staticmethod
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS'] def error(message):
for folder in folders: print >> stderr, message
if not isabs(default[folder]): exit()
def overwrite_environment(self):
"""Overwrite default values from the environment."""
for key in environ.keys():
if key in self.default.keys():
self.default[key] = environ[key]
def check_settings(self):
"""Perform various checking."""
# Folders
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS']
for folder in folders:
if not isabs(self.default[folder]):
# Get the absolute path. # Get the absolute path.
default[folder] = abspath(default[folder]) self.default[folder] = abspath(self.default[folder])
# Test the folder # Test the folder
if not exists(default[folder]): if not exists(self.default[folder]):
print >> stderr, 'The folder %s does not exist.' % default[folder] msg = 'The folder %s does not exist.' % self.default[folder]
exit() self.error(msg)
# Test files # Test files
osm_file = None for f in listdir(self.default['SETTINGS']):
for f in listdir(default['SETTINGS']):
if f.endswith('.pbf'): if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f) self.osm_file = join(self.default['SETTINGS'], f)
""" if not self.osm_file:
# Todo : need fix custom URL and sporadic diff : daily, hourly and minutely msg = 'OSM file *.osm.pbf is missing in %s' % self.default['SETTINGS']
if f == 'custom_url_diff.txt': self.error(msg)
with open(join(default['SETTINGS'], f), 'r') as content_file:
default['BASE_URL'] = content_file.read()
"""
if not osm_file: self.info('The checkup is OK. The container will continue soon, after the database.')
print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS'] sleep(45)
exit()
# In docker-compose, we should wait for the DB is ready. def _check_latest_timestamp(self):
print 'The checkup is OK. The container will continue soon, after the database.' """Fetch the latest timestamp."""
sleep(45)
# Finally launch the listening process.
while True:
# Check if diff to be imported is empty. If not, take the latest diff. # Check if diff to be imported is empty. If not, take the latest diff.
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE'])) diff_to_be_imported = sorted(listdir(self.default['IMPORT_QUEUE']))
if len(diff_to_be_imported): if len(diff_to_be_imported):
file_name = diff_to_be_imported[-1].split('.')[0] file_name = diff_to_be_imported[-1].split('.')[0]
timestamp = file_name.split('->-')[1] timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest not imported diff : %s' % timestamp self.info('Timestamp from the latest not imported diff : %s' % timestamp)
else: else:
# Check if imported diff is empty. If not, take the latest diff. # Check if imported diff is empty. If not, take the latest diff.
imported_diff = sorted(listdir(default['IMPORT_DONE'])) imported_diff = sorted(listdir(self.default['IMPORT_DONE']))
if len(imported_diff): if len(imported_diff):
file_name = imported_diff[-1].split('.')[0] file_name = imported_diff[-1].split('.')[0]
timestamp = file_name.split('->-')[1] timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest imported diff : %s' % timestamp self.info('Timestamp from the latest imported diff : %s' % timestamp)
else: else:
# Take the timestamp from original file. # Take the timestamp from original file.
command = ['osmconvert', osm_file, '--out-timestamp'] command = ['osmconvert', self.osm_file, '--out-timestamp']
processus = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) processus = Popen(
command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
timestamp, err = processus.communicate() timestamp, err = processus.communicate()
# Remove new line # Remove new line
timestamp = timestamp.strip() timestamp = timestamp.strip()
print 'Timestamp from the original state file : %s' % timestamp self.info('Timestamp from the original state file : %s' % timestamp)
# Removing some \ in the timestamp. # Removing some \ in the timestamp.
timestamp = timestamp.replace('\\', '') timestamp = timestamp.replace('\\', '')
return timestamp
def download(self):
"""Infinite loop to download diff files on a regular interval."""
while True:
timestamp = self._check_latest_timestamp()
# Save time # Save time
current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
print 'Old time : %s' % timestamp self.info('Old time : %s' % timestamp)
print 'Current time : %s' % current_time self.info('Current time : %s' % current_time)
# Destination # Destination
file_name = '%s->-%s.osc.gz' % (timestamp, current_time) file_name = '%s->-%s.osc.gz' % (timestamp, current_time)
file_path = join(default['IMPORT_QUEUE'], file_name) file_path = join(self.default['IMPORT_QUEUE'], file_name)
# Command # Command
command = ['osmupdate', '-v'] command = ['osmupdate', '-v']
command += ['--max-days=' + default['MAX_DAYS']] command += ['--max-days=' + self.default['MAX_DAYS']]
command += [default['DIFF']] command += [self.default['DIFF']]
command += ['--max-merge=' + default['MAX_MERGE']] command += ['--max-merge=' + self.default['MAX_MERGE']]
command += ['--compression-level=' + default['COMPRESSION_LEVEL']] command += ['--compression-level=' + self.default['COMPRESSION_LEVEL']]
command += ['--base-url=' + default['BASE_URL']] command += ['--base-url=' + self.default['BASE_URL']]
command.append(timestamp) command.append(timestamp)
command.append(file_path) command.append(file_path)
print ' '.join(command) self.info(' '.join(command))
if call(command) != 0: if call(command) != 0:
print >> stderr, 'An error occured in osmupdate. Let\'s try again.' self.info('An error occured in osmupdate. Let\'s try again.')
# Sleep less. # Sleep less.
print 'Sleeping for 2 seconds.' self.info('Sleeping for 2 seconds.')
sleep(2.0) sleep(2.0)
else: else:
# Everything was fine, let's sleeping. # Everything was fine, let's sleeping.
print 'Sleeping for %s seconds.' % default['TIME'] self.info('Sleeping for %s seconds.' % self.default['TIME'])
sleep(float(default['TIME'])) sleep(float(self.default['TIME']))
if __name__ == '__main__':
downloader = Downloader()
downloader.overwrite_environment()
downloader.check_settings()
downloader.download()