mirror
/
docker-osm
kopia lustrzana https://github.com/kartoza/docker-osm
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Merge pull request #28 from Gustry/refactoring 

Refactoring osmupdate and imposm
pull/31/head
Etienne Trimaille 2015-12-29 14:10:23 +01:00
rodzic 4c1899854f eb67fae10f
commit 86340505c6
2 zmienionych plików z 331 dodań i 264 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

391
docker-imposm3/importer.py
Wyświetl plik

@ -28,211 +28,256 @@ from subprocess import call
from time import sleep from time import sleep
from sys import stderr from sys import stderr
# All these default values can be overwritten by env vars
default = {
'TIME': 120,
'USER': 'docker',
'PASSWORD': 'docker',
'DATABASE': 'gis',
'HOST': 'db',
'PORT': '5432',
'SETTINGS': 'settings',
'CACHE': 'cache',
'IMPORT_DONE': 'import_done',
'IMPORT_QUEUE': 'import_queue',
'SRID': '4326',
'OPTIMIZE': 'false',
'DBSCHEMA_PRODUCTION': 'public',
'DBSCHEMA_IMPORT': 'import',
'DBSCHEMA_BACKUP': 'backup',
'QGIS_STYLE': 'yes'
}
# Check if we overwrite default values. class Importer(object):
for key in environ.keys():
if key in default.keys():
default[key] = environ[key]
# Check valid SRID. def __init__(self):
if default['SRID'] not in ['4326', '3857']: # Default values which can be overwritten.
print >> stderr, 'SRID not supported : %s' % default['SRID'] self.default = {
exit() 'TIME': 120,
'USER': 'docker',
'PASSWORD': 'docker',
'DATABASE': 'gis',
'HOST': 'db',
'PORT': '5432',
'SETTINGS': 'settings',
'CACHE': 'cache',
'IMPORT_DONE': 'import_done',
'IMPORT_QUEUE': 'import_queue',
'SRID': '4326',
'OPTIMIZE': 'false',
'DBSCHEMA_PRODUCTION': 'public',
'DBSCHEMA_IMPORT': 'import',
'DBSCHEMA_BACKUP': 'backup',
'QGIS_STYLE': 'yes'
}
self.osm_file = None
self.mapping_file = None
self.post_import_file = None
self.qgis_style = None
# Check valid QGIS_STYLE. self.cursor = None
if default['QGIS_STYLE'] not in ['yes', 'no']: self.postgis_uri = None
print >> stderr, 'QGIS_STYLE not supported : %s' % default['QGIS_STYLE']
exit()
# Check folders. @staticmethod
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE'] def info(message):
for folder in folders: print message
if not isabs(default[folder]):
# Get the absolute path.
default[folder] = abspath(default[folder])
# Test the folder @staticmethod
if not exists(default[folder]): def error(message):
print >> stderr, 'The folder %s does not exist.' % default[folder] print >> stderr, message
exit() exit()
# Test files def overwrite_environment(self):
osm_file = None """Overwrite default values from the environment."""
mapping_file = None for key in environ.keys():
post_import_file = None if key in self.default.keys():
qgis_style = None self.default[key] = environ[key]
for f in listdir(default['SETTINGS']):
if f.endswith('.pbf'): def check_settings(self):
osm_file = join(default['SETTINGS'], f) """Perform various checking."""
if f.endswith('.json'): # Check valid SRID.
mapping_file = join(default['SETTINGS'], f) if self.default['SRID'] not in ['4326', '3857']:
msg = 'SRID not supported : %s' % self.default['SRID']
self.error(msg)
if f == 'post-pbf-import.sql': # Check valid QGIS_STYLE.
post_import_file = join(default['SETTINGS'], f) if self.default['QGIS_STYLE'] not in ['yes', 'no']:
msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
self.error(msg)
if f == 'qgis_style.sql': # Check folders.
qgis_style = join(default['SETTINGS'], f) folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
for folder in folders:
if not isabs(self.default[folder]):
# Get the absolute path.
self.default[folder] = abspath(self.default[folder])
if not osm_file: # Test the folder
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS'] if not exists(self.default[folder]):
exit() msg = 'The folder %s does not exist.' % self.default[folder]
self.error(msg)
if not mapping_file: # Test files
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS'] for f in listdir(self.default['SETTINGS']):
exit()
if not post_import_file: if f.endswith('.pbf'):
print 'No *.sql detected in %s' % default['SETTINGS'] self.osm_file = join(self.default['SETTINGS'], f)
else:
print '%s detected for post import.' % post_import_file
if not qgis_style and default['QGIS_STYLE'] == 'yes': if f.endswith('.json'):
print >> stderr, 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % default['SETTINGS'] self.mapping_file = join(self.default['SETTINGS'], f)
exit()
elif qgis_style and default['QGIS_STYLE']:
print '%s detected for QGIS styling.' % qgis_style
else:
print 'Not using QGIS default styles.'
# Create the timestamp file if f == 'post-pbf-import.sql':
file_path = join(default['SETTINGS'], 'timestamp.txt') self.post_import_file = join(self.default['SETTINGS'], f)
timestamp_file = open(file_path, 'w')
timestamp_file.write('UNDEFINED\n')
timestamp_file.close()
# In docker-compose, we should wait for the DB is ready. if f == 'qgis_style.sql':
print 'The checkup is OK. The container will continue soon, after the database.' self.qgis_style = join(self.default['SETTINGS'], f)
sleep(45)
# Check postgis. if not self.osm_file:
try: msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
connection = connect( self.error(msg)
"dbname='%s' user='%s' host='%s' password='%s'" % (
default['DATABASE'],
default['USER'],
default['HOST'],
default['PASSWORD']))
cursor = connection.cursor()
except OperationalError as e:
print >> stderr, e
exit()
postgis_uri = 'postgis://%s:%s@%s/%s' % ( if not self.mapping_file:
default['USER'], msg = 'Mapping file *.json is missing in %s' % self.default['SETTINGS']
default['PASSWORD'], self.error(msg)
default['HOST'],
default['DATABASE'])
if not self.post_import_file:
self.info('No *.sql detected in %s' % self.default['SETTINGS'])
else:
self.info('%s detected for post import.' % self.post_import_file)
if not self.qgis_style and self.default['QGIS_STYLE'] == 'yes':
msg = 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % self.default['SETTINGS']
self.error(msg)
elif self.qgis_style and self.default['QGIS_STYLE']:
self.info('%s detected for QGIS styling.' % self.qgis_style)
else:
self.info('Not using QGIS default styles.')
# Check if there is a table starting with 'osm_' # In docker-compose, we should wait for the DB is ready.
sql = 'select count(*) ' \ self.info('The checkup is OK. The container will continue soon, after the database.')
'from information_schema.tables ' \ sleep(45)
'where table_name like \'osm_%\';'
# noinspection PyUnboundLocalVariable
cursor.execute(sql)
osm_tables = cursor.fetchone()[0]
if osm_tables < 1:
# It means that the DB is empty. Let's import the PBF file.
command = ['imposm3', 'import', '-diff', '-deployproduction']
command += ['-overwritecache', '-cachedir', default['CACHE']]
command += ['-srid', default['SRID']]
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
command += ['-diffdir', default['SETTINGS']]
command += ['-mapping', mapping_file]
command += ['-read', osm_file]
command += ['-write', '-connection', postgis_uri]
print 'The database is empty. Let\'s import the PBF : %s' % osm_file def create_timestamp(self):
print ' '.join(command) file_path = join(self.default['SETTINGS'], 'timestamp.txt')
if not call(command) == 0: timestamp_file = open(file_path, 'w')
print >> stderr, 'An error occured in imposm with the original file.' timestamp_file.write('UNDEFINED\n')
exit() timestamp_file.close()
else:
print 'Import PBF successful : %s' % osm_file
if post_import_file or qgis_style: def update_timestamp(self, database_timestamp):
# Set the password for psql file_path = join(self.default['SETTINGS'], 'timestamp.txt')
environ['PGPASSWORD'] = default['PASSWORD'] timestamp_file = open(file_path, 'w')
timestamp_file.write('%s\n' % database_timestamp)
timestamp_file.close()
if post_import_file: def check_postgis(self):
print 'Running the post import SQL file.' try:
connection = connect(
"dbname='%s' user='%s' host='%s' password='%s'" % (
self.default['DATABASE'],
self.default['USER'],
self.default['HOST'],
self.default['PASSWORD']))
self.cursor = connection.cursor()
except OperationalError as e:
print >> stderr, e
exit()
self.postgis_uri = 'postgis://%s:%s@%s/%s' % (
self.default['USER'],
self.default['PASSWORD'],
self.default['HOST'],
self.default['DATABASE'])
def import_custom_sql(self):
self.info('Running the post import SQL file.')
command = ['psql'] command = ['psql']
command += ['-h', default['HOST']] command += ['-h', self.default['HOST']]
command += ['-U', default['USER']] command += ['-U', self.default['USER']]
command += ['-d', default['DATABASE']] command += ['-d', self.default['DATABASE']]
command += ['-f', post_import_file] command += ['-f', self.post_import_file]
call(command) call(command)
if qgis_style: def import_qgis_styles(self):
'Installing QGIS styles.' self.info('Installing QGIS styles.')
command = ['psql'] command = ['psql']
command += ['-h', default['HOST']] command += ['-h', self.default['HOST']]
command += ['-U', default['USER']] command += ['-U', self.default['USER']]
command += ['-d', default['DATABASE']] command += ['-d', self.default['DATABASE']]
command += ['-f', qgis_style] command += ['-f', self.qgis_style]
call(command) call(command)
else:
print 'The database is not empty. Let\'s import only diff files.'
# Finally launch the listening process. def count_table(self, name):
while True: """Check if there is a table starting with name."""
import_queue = sorted(listdir(default['IMPORT_QUEUE'])) sql = 'select count(*) ' \
if len(import_queue) > 0: 'from information_schema.tables ' \
for diff in import_queue: 'where table_name like \'%s\';' % name
print 'Importing diff %s' % diff # noinspection PyUnboundLocalVariable
command = ['imposm3', 'diff'] self.cursor.execute(sql)
command += ['-cachedir', default['CACHE']] return self.cursor.fetchone()[0]
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
command += ['-srid', default['SRID']]
command += ['-diffdir', default['SETTINGS']]
command += ['-mapping', mapping_file]
command += ['-connection', postgis_uri]
command += [join(default['IMPORT_QUEUE'], diff)]
print ' '.join(command) def run(self):
if call(command) == 0: osm_tables = self.count_table('osm_%')
move( if osm_tables < 1:
join(default['IMPORT_QUEUE'], diff), # It means that the DB is empty. Let's import the PBF file.
join(default['IMPORT_DONE'], diff)) self._first_pbf_import()
else:
self.info('The database is not empty. Let\'s import only diff files.')
# Update the timestamp in the file. self._import_diff()
database_timestamp = diff.split('.')[0].split('->-')[1]
file_path = join(default['SETTINGS'], 'timestamp.txt')
timestamp_file = open(file_path, 'w')
timestamp_file.write('%s\n' % database_timestamp)
timestamp_file.close()
print 'Import diff successful : %s' % diff def _first_pbf_import(self):
else: command = ['imposm3', 'import', '-diff', '-deployproduction']
print >> stderr, 'An error occured in imposm with a diff.' command += ['-overwritecache', '-cachedir', self.default['CACHE']]
exit() command += ['-srid', self.default['SRID']]
command += ['-dbschema-production',
self.default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', self.default['DBSCHEMA_BACKUP']]
command += ['-diffdir', self.default['SETTINGS']]
command += ['-mapping', self.mapping_file]
command += ['-read', self.osm_file]
command += ['-write', '-connection', self.postgis_uri]
self.info('The database is empty. Let\'s import the PBF : %s' % self.osm_file)
self.info(' '.join(command))
if not call(command) == 0:
msg = 'An error occured in imposm with the original file.'
self.error(msg)
else:
self.info('Import PBF successful : %s' % self.osm_file)
if len(listdir(default['IMPORT_QUEUE'])) == 0: if self.post_import_file or self.qgis_style:
print 'Sleeping for %s seconds.' % default['TIME'] # Set the password for psql
sleep(float(default['TIME'])) environ['PGPASSWORD'] = self.default['PASSWORD']
if self.post_import_file:
self.import_custom_sql()
if self.qgis_style:
self.import_qgis_styles()
def _import_diff(self):
# Finally launch the listening process.
while True:
import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
if len(import_queue) > 0:
for diff in import_queue:
self.info('Importing diff %s' % diff)
command = ['imposm3', 'diff']
command += ['-cachedir', self.default['CACHE']]
command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', self.default['DBSCHEMA_BACKUP']]
command += ['-srid', self.default['SRID']]
command += ['-diffdir', self.default['SETTINGS']]
command += ['-mapping', self.mapping_file]
command += ['-connection', self.postgis_uri]
command += [join(self.default['IMPORT_QUEUE'], diff)]
self.info(' '.join(command))
if call(command) == 0:
move(
join(self.default['IMPORT_QUEUE'], diff),
join(self.default['IMPORT_DONE'], diff))
# Update the timestamp in the file.
database_timestamp = diff.split('.')[0].split('->-')[1]
self.update_timestamp(database_timestamp)
self.info('Import diff successful : %s' % diff)
else:
msg = 'An error occured in imposm with a diff.'
self.error(msg)
if len(listdir(self.default['IMPORT_QUEUE'])) == 0:
self.info('Sleeping for %s seconds.' % self.default['TIME'])
sleep(float(self.default['TIME']))
if __name__ == '__main__':
importer = Importer()
importer.overwrite_environment()
importer.check_settings()
importer.create_timestamp()
importer.check_postgis()
importer.run()

204
docker-osmupdate/download.py
Wyświetl plik

@ -27,113 +27,135 @@ from datetime import datetime
from time import sleep from time import sleep
from sys import stderr from sys import stderr
# Default values which can be overwritten.
default = {
'MAX_DAYS': '100',
'DIFF': 'sporadic',
'MAX_MERGE': '7',
'COMPRESSION_LEVEL': '1',
'BASE_URL': 'http://planet.openstreetmap.org/replication/',
'IMPORT_QUEUE': 'import_queue',
'IMPORT_DONE': 'import_done',
'SETTINGS': 'settings',
'TIME': 120,
}
for key in environ.keys(): class Downloader(object):
if key in default.keys():
default[key] = environ[key]
# Folders def __init__(self):
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS'] # Default values which can be overwritten.
for folder in folders: self.default = {
if not isabs(default[folder]): 'MAX_DAYS': '100',
# Get the absolute path. 'DIFF': 'sporadic',
default[folder] = abspath(default[folder]) 'MAX_MERGE': '7',
'COMPRESSION_LEVEL': '1',
'BASE_URL': 'http://planet.openstreetmap.org/replication/',
'IMPORT_QUEUE': 'import_queue',
'IMPORT_DONE': 'import_done',
'SETTINGS': 'settings',
'TIME': 120,
}
self.osm_file = None
# Test the folder @staticmethod
if not exists(default[folder]): def info(message):
print >> stderr, 'The folder %s does not exist.' % default[folder] print message
@staticmethod
def error(message):
print >> stderr, message
exit() exit()
# Test files def overwrite_environment(self):
osm_file = None """Overwrite default values from the environment."""
for f in listdir(default['SETTINGS']): for key in environ.keys():
if key in self.default.keys():
self.default[key] = environ[key]
if f.endswith('.pbf'): def check_settings(self):
osm_file = join(default['SETTINGS'], f) """Perform various checking."""
# Folders
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS']
for folder in folders:
if not isabs(self.default[folder]):
# Get the absolute path.
self.default[folder] = abspath(self.default[folder])
""" # Test the folder
# Todo : need fix custom URL and sporadic diff : daily, hourly and minutely if not exists(self.default[folder]):
if f == 'custom_url_diff.txt': msg = 'The folder %s does not exist.' % self.default[folder]
with open(join(default['SETTINGS'], f), 'r') as content_file: self.error(msg)
default['BASE_URL'] = content_file.read()
"""
if not osm_file: # Test files
print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS'] for f in listdir(self.default['SETTINGS']):
exit()
# In docker-compose, we should wait for the DB is ready. if f.endswith('.pbf'):
print 'The checkup is OK. The container will continue soon, after the database.' self.osm_file = join(self.default['SETTINGS'], f)
sleep(45)
# Finally launch the listening process. if not self.osm_file:
while True: msg = 'OSM file *.osm.pbf is missing in %s' % self.default['SETTINGS']
# Check if diff to be imported is empty. If not, take the latest diff. self.error(msg)
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
if len(diff_to_be_imported): self.info('The checkup is OK. The container will continue soon, after the database.')
file_name = diff_to_be_imported[-1].split('.')[0] sleep(45)
timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest not imported diff : %s' % timestamp def _check_latest_timestamp(self):
else: """Fetch the latest timestamp."""
# Check if imported diff is empty. If not, take the latest diff. # Check if diff to be imported is empty. If not, take the latest diff.
imported_diff = sorted(listdir(default['IMPORT_DONE'])) diff_to_be_imported = sorted(listdir(self.default['IMPORT_QUEUE']))
if len(imported_diff): if len(diff_to_be_imported):
file_name = imported_diff[-1].split('.')[0] file_name = diff_to_be_imported[-1].split('.')[0]
timestamp = file_name.split('->-')[1] timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest imported diff : %s' % timestamp self.info('Timestamp from the latest not imported diff : %s' % timestamp)
else: else:
# Take the timestamp from original file. # Check if imported diff is empty. If not, take the latest diff.
command = ['osmconvert', osm_file, '--out-timestamp'] imported_diff = sorted(listdir(self.default['IMPORT_DONE']))
processus = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) if len(imported_diff):
timestamp, err = processus.communicate() file_name = imported_diff[-1].split('.')[0]
timestamp = file_name.split('->-')[1]
self.info('Timestamp from the latest imported diff : %s' % timestamp)
# Remove new line else:
timestamp = timestamp.strip() # Take the timestamp from original file.
command = ['osmconvert', self.osm_file, '--out-timestamp']
processus = Popen(
command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
timestamp, err = processus.communicate()
print 'Timestamp from the original state file : %s' % timestamp # Remove new line
timestamp = timestamp.strip()
# Removing some \ in the timestamp. self.info('Timestamp from the original state file : %s' % timestamp)
timestamp = timestamp.replace('\\', '')
# Save time # Removing some \ in the timestamp.
current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') timestamp = timestamp.replace('\\', '')
print 'Old time : %s' % timestamp return timestamp
print 'Current time : %s' % current_time
# Destination def download(self):
file_name = '%s->-%s.osc.gz' % (timestamp, current_time) """Infinite loop to download diff files on a regular interval."""
file_path = join(default['IMPORT_QUEUE'], file_name) while True:
timestamp = self._check_latest_timestamp()
# Command # Save time
command = ['osmupdate', '-v'] current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
command += ['--max-days=' + default['MAX_DAYS']] self.info('Old time : %s' % timestamp)
command += [default['DIFF']] self.info('Current time : %s' % current_time)
command += ['--max-merge=' + default['MAX_MERGE']]
command += ['--compression-level=' + default['COMPRESSION_LEVEL']]
command += ['--base-url=' + default['BASE_URL']]
command.append(timestamp)
command.append(file_path)
print ' '.join(command) # Destination
if call(command) != 0: file_name = '%s->-%s.osc.gz' % (timestamp, current_time)
print >> stderr, 'An error occured in osmupdate. Let\'s try again.' file_path = join(self.default['IMPORT_QUEUE'], file_name)
# Sleep less.
print 'Sleeping for 2 seconds.' # Command
sleep(2.0) command = ['osmupdate', '-v']
else: command += ['--max-days=' + self.default['MAX_DAYS']]
# Everything was fine, let's sleeping. command += [self.default['DIFF']]
print 'Sleeping for %s seconds.' % default['TIME'] command += ['--max-merge=' + self.default['MAX_MERGE']]
sleep(float(default['TIME'])) command += ['--compression-level=' + self.default['COMPRESSION_LEVEL']]
command += ['--base-url=' + self.default['BASE_URL']]
command.append(timestamp)
command.append(file_path)
self.info(' '.join(command))
if call(command) != 0:
self.info('An error occured in osmupdate. Let\'s try again.')
# Sleep less.
self.info('Sleeping for 2 seconds.')
sleep(2.0)
else:
# Everything was fine, let's sleeping.
self.info('Sleeping for %s seconds.' % self.default['TIME'])
sleep(float(self.default['TIME']))
if __name__ == '__main__':
downloader = Downloader()
downloader.overwrite_environment()
downloader.check_settings()
downloader.download()