kopia lustrzana https://github.com/kartoza/docker-osm
refactoring imposm
Etienne Trimaille
rodzic
c3692553c2
commit
eb67fae10f
|
|
@ -28,211 +28,256 @@ from subprocess import call
|
||||||
from time import sleep
|
from time import sleep
|
||||||
from sys import stderr
|
from sys import stderr
|
||||||
|
|
||||||
# All these default values can be overwritten by env vars
|
|
||||||
default = {
|
|
||||||
'TIME': 120,
|
|
||||||
'USER': 'docker',
|
|
||||||
'PASSWORD': 'docker',
|
|
||||||
'DATABASE': 'gis',
|
|
||||||
'HOST': 'db',
|
|
||||||
'PORT': '5432',
|
|
||||||
'SETTINGS': 'settings',
|
|
||||||
'CACHE': 'cache',
|
|
||||||
'IMPORT_DONE': 'import_done',
|
|
||||||
'IMPORT_QUEUE': 'import_queue',
|
|
||||||
'SRID': '4326',
|
|
||||||
'OPTIMIZE': 'false',
|
|
||||||
'DBSCHEMA_PRODUCTION': 'public',
|
|
||||||
'DBSCHEMA_IMPORT': 'import',
|
|
||||||
'DBSCHEMA_BACKUP': 'backup',
|
|
||||||
'QGIS_STYLE': 'yes'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if we overwrite default values.
|
class Importer(object):
|
||||||
for key in environ.keys():
|
|
||||||
if key in default.keys():
|
|
||||||
default[key] = environ[key]
|
|
||||||
|
|
||||||
# Check valid SRID.
|
def __init__(self):
|
||||||
if default['SRID'] not in ['4326', '3857']:
|
# Default values which can be overwritten.
|
||||||
print >> stderr, 'SRID not supported : %s' % default['SRID']
|
self.default = {
|
||||||
exit()
|
'TIME': 120,
|
||||||
|
'USER': 'docker',
|
||||||
|
'PASSWORD': 'docker',
|
||||||
|
'DATABASE': 'gis',
|
||||||
|
'HOST': 'db',
|
||||||
|
'PORT': '5432',
|
||||||
|
'SETTINGS': 'settings',
|
||||||
|
'CACHE': 'cache',
|
||||||
|
'IMPORT_DONE': 'import_done',
|
||||||
|
'IMPORT_QUEUE': 'import_queue',
|
||||||
|
'SRID': '4326',
|
||||||
|
'OPTIMIZE': 'false',
|
||||||
|
'DBSCHEMA_PRODUCTION': 'public',
|
||||||
|
'DBSCHEMA_IMPORT': 'import',
|
||||||
|
'DBSCHEMA_BACKUP': 'backup',
|
||||||
|
'QGIS_STYLE': 'yes'
|
||||||
|
}
|
||||||
|
self.osm_file = None
|
||||||
|
self.mapping_file = None
|
||||||
|
self.post_import_file = None
|
||||||
|
self.qgis_style = None
|
||||||
|
|
||||||
# Check valid QGIS_STYLE.
|
self.cursor = None
|
||||||
if default['QGIS_STYLE'] not in ['yes', 'no']:
|
self.postgis_uri = None
|
||||||
print >> stderr, 'QGIS_STYLE not supported : %s' % default['QGIS_STYLE']
|
|
||||||
exit()
|
|
||||||
|
|
||||||
# Check folders.
|
@staticmethod
|
||||||
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
|
def info(message):
|
||||||
for folder in folders:
|
print message
|
||||||
if not isabs(default[folder]):
|
|
||||||
# Get the absolute path.
|
|
||||||
default[folder] = abspath(default[folder])
|
|
||||||
|
|
||||||
# Test the folder
|
@staticmethod
|
||||||
if not exists(default[folder]):
|
def error(message):
|
||||||
print >> stderr, 'The folder %s does not exist.' % default[folder]
|
print >> stderr, message
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
# Test files
|
def overwrite_environment(self):
|
||||||
osm_file = None
|
"""Overwrite default values from the environment."""
|
||||||
mapping_file = None
|
for key in environ.keys():
|
||||||
post_import_file = None
|
if key in self.default.keys():
|
||||||
qgis_style = None
|
self.default[key] = environ[key]
|
||||||
for f in listdir(default['SETTINGS']):
|
|
||||||
|
|
||||||
if f.endswith('.pbf'):
|
def check_settings(self):
|
||||||
osm_file = join(default['SETTINGS'], f)
|
"""Perform various checking."""
|
||||||
|
|
||||||
if f.endswith('.json'):
|
# Check valid SRID.
|
||||||
mapping_file = join(default['SETTINGS'], f)
|
if self.default['SRID'] not in ['4326', '3857']:
|
||||||
|
msg = 'SRID not supported : %s' % self.default['SRID']
|
||||||
|
self.error(msg)
|
||||||
|
|
||||||
if f == 'post-pbf-import.sql':
|
# Check valid QGIS_STYLE.
|
||||||
post_import_file = join(default['SETTINGS'], f)
|
if self.default['QGIS_STYLE'] not in ['yes', 'no']:
|
||||||
|
msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
|
||||||
|
self.error(msg)
|
||||||
|
|
||||||
if f == 'qgis_style.sql':
|
# Check folders.
|
||||||
qgis_style = join(default['SETTINGS'], f)
|
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
|
||||||
|
for folder in folders:
|
||||||
|
if not isabs(self.default[folder]):
|
||||||
|
# Get the absolute path.
|
||||||
|
self.default[folder] = abspath(self.default[folder])
|
||||||
|
|
||||||
if not osm_file:
|
# Test the folder
|
||||||
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
|
if not exists(self.default[folder]):
|
||||||
exit()
|
msg = 'The folder %s does not exist.' % self.default[folder]
|
||||||
|
self.error(msg)
|
||||||
|
|
||||||
if not mapping_file:
|
# Test files
|
||||||
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
|
for f in listdir(self.default['SETTINGS']):
|
||||||
exit()
|
|
||||||
|
|
||||||
if not post_import_file:
|
if f.endswith('.pbf'):
|
||||||
print 'No *.sql detected in %s' % default['SETTINGS']
|
self.osm_file = join(self.default['SETTINGS'], f)
|
||||||
else:
|
|
||||||
print '%s detected for post import.' % post_import_file
|
|
||||||
|
|
||||||
if not qgis_style and default['QGIS_STYLE'] == 'yes':
|
if f.endswith('.json'):
|
||||||
print >> stderr, 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % default['SETTINGS']
|
self.mapping_file = join(self.default['SETTINGS'], f)
|
||||||
exit()
|
|
||||||
elif qgis_style and default['QGIS_STYLE']:
|
|
||||||
print '%s detected for QGIS styling.' % qgis_style
|
|
||||||
else:
|
|
||||||
print 'Not using QGIS default styles.'
|
|
||||||
|
|
||||||
# Create the timestamp file
|
if f == 'post-pbf-import.sql':
|
||||||
file_path = join(default['SETTINGS'], 'timestamp.txt')
|
self.post_import_file = join(self.default['SETTINGS'], f)
|
||||||
timestamp_file = open(file_path, 'w')
|
|
||||||
timestamp_file.write('UNDEFINED\n')
|
|
||||||
timestamp_file.close()
|
|
||||||
|
|
||||||
# In docker-compose, we should wait for the DB is ready.
|
if f == 'qgis_style.sql':
|
||||||
print 'The checkup is OK. The container will continue soon, after the database.'
|
self.qgis_style = join(self.default['SETTINGS'], f)
|
||||||
sleep(45)
|
|
||||||
|
|
||||||
# Check postgis.
|
if not self.osm_file:
|
||||||
try:
|
msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
|
||||||
connection = connect(
|
self.error(msg)
|
||||||
"dbname='%s' user='%s' host='%s' password='%s'" % (
|
|
||||||
default['DATABASE'],
|
|
||||||
default['USER'],
|
|
||||||
default['HOST'],
|
|
||||||
default['PASSWORD']))
|
|
||||||
cursor = connection.cursor()
|
|
||||||
except OperationalError as e:
|
|
||||||
print >> stderr, e
|
|
||||||
exit()
|
|
||||||
|
|
||||||
postgis_uri = 'postgis://%s:%s@%s/%s' % (
|
if not self.mapping_file:
|
||||||
default['USER'],
|
msg = 'Mapping file *.json is missing in %s' % self.default['SETTINGS']
|
||||||
default['PASSWORD'],
|
self.error(msg)
|
||||||
default['HOST'],
|
|
||||||
default['DATABASE'])
|
|
||||||
|
|
||||||
|
if not self.post_import_file:
|
||||||
|
self.info('No *.sql detected in %s' % self.default['SETTINGS'])
|
||||||
|
else:
|
||||||
|
self.info('%s detected for post import.' % self.post_import_file)
|
||||||
|
|
||||||
|
if not self.qgis_style and self.default['QGIS_STYLE'] == 'yes':
|
||||||
|
msg = 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % self.default['SETTINGS']
|
||||||
|
self.error(msg)
|
||||||
|
elif self.qgis_style and self.default['QGIS_STYLE']:
|
||||||
|
self.info('%s detected for QGIS styling.' % self.qgis_style)
|
||||||
|
else:
|
||||||
|
self.info('Not using QGIS default styles.')
|
||||||
|
|
||||||
# Check if there is a table starting with 'osm_'
|
# In docker-compose, we should wait for the DB is ready.
|
||||||
sql = 'select count(*) ' \
|
self.info('The checkup is OK. The container will continue soon, after the database.')
|
||||||
'from information_schema.tables ' \
|
sleep(45)
|
||||||
'where table_name like \'osm_%\';'
|
|
||||||
# noinspection PyUnboundLocalVariable
|
|
||||||
cursor.execute(sql)
|
|
||||||
osm_tables = cursor.fetchone()[0]
|
|
||||||
if osm_tables < 1:
|
|
||||||
# It means that the DB is empty. Let's import the PBF file.
|
|
||||||
command = ['imposm3', 'import', '-diff', '-deployproduction']
|
|
||||||
command += ['-overwritecache', '-cachedir', default['CACHE']]
|
|
||||||
command += ['-srid', default['SRID']]
|
|
||||||
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
|
||||||
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
|
||||||
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
|
||||||
command += ['-diffdir', default['SETTINGS']]
|
|
||||||
command += ['-mapping', mapping_file]
|
|
||||||
command += ['-read', osm_file]
|
|
||||||
command += ['-write', '-connection', postgis_uri]
|
|
||||||
|
|
||||||
print 'The database is empty. Let\'s import the PBF : %s' % osm_file
|
def create_timestamp(self):
|
||||||
print ' '.join(command)
|
file_path = join(self.default['SETTINGS'], 'timestamp.txt')
|
||||||
if not call(command) == 0:
|
timestamp_file = open(file_path, 'w')
|
||||||
print >> stderr, 'An error occured in imposm with the original file.'
|
timestamp_file.write('UNDEFINED\n')
|
||||||
exit()
|
timestamp_file.close()
|
||||||
else:
|
|
||||||
print 'Import PBF successful : %s' % osm_file
|
|
||||||
|
|
||||||
if post_import_file or qgis_style:
|
def update_timestamp(self, database_timestamp):
|
||||||
# Set the password for psql
|
file_path = join(self.default['SETTINGS'], 'timestamp.txt')
|
||||||
environ['PGPASSWORD'] = default['PASSWORD']
|
timestamp_file = open(file_path, 'w')
|
||||||
|
timestamp_file.write('%s\n' % database_timestamp)
|
||||||
|
timestamp_file.close()
|
||||||
|
|
||||||
if post_import_file:
|
def check_postgis(self):
|
||||||
print 'Running the post import SQL file.'
|
try:
|
||||||
|
connection = connect(
|
||||||
|
"dbname='%s' user='%s' host='%s' password='%s'" % (
|
||||||
|
self.default['DATABASE'],
|
||||||
|
self.default['USER'],
|
||||||
|
self.default['HOST'],
|
||||||
|
self.default['PASSWORD']))
|
||||||
|
self.cursor = connection.cursor()
|
||||||
|
except OperationalError as e:
|
||||||
|
print >> stderr, e
|
||||||
|
exit()
|
||||||
|
|
||||||
|
self.postgis_uri = 'postgis://%s:%s@%s/%s' % (
|
||||||
|
self.default['USER'],
|
||||||
|
self.default['PASSWORD'],
|
||||||
|
self.default['HOST'],
|
||||||
|
self.default['DATABASE'])
|
||||||
|
|
||||||
|
def import_custom_sql(self):
|
||||||
|
self.info('Running the post import SQL file.')
|
||||||
command = ['psql']
|
command = ['psql']
|
||||||
command += ['-h', default['HOST']]
|
command += ['-h', self.default['HOST']]
|
||||||
command += ['-U', default['USER']]
|
command += ['-U', self.default['USER']]
|
||||||
command += ['-d', default['DATABASE']]
|
command += ['-d', self.default['DATABASE']]
|
||||||
command += ['-f', post_import_file]
|
command += ['-f', self.post_import_file]
|
||||||
call(command)
|
call(command)
|
||||||
|
|
||||||
if qgis_style:
|
def import_qgis_styles(self):
|
||||||
'Installing QGIS styles.'
|
self.info('Installing QGIS styles.')
|
||||||
command = ['psql']
|
command = ['psql']
|
||||||
command += ['-h', default['HOST']]
|
command += ['-h', self.default['HOST']]
|
||||||
command += ['-U', default['USER']]
|
command += ['-U', self.default['USER']]
|
||||||
command += ['-d', default['DATABASE']]
|
command += ['-d', self.default['DATABASE']]
|
||||||
command += ['-f', qgis_style]
|
command += ['-f', self.qgis_style]
|
||||||
call(command)
|
call(command)
|
||||||
else:
|
|
||||||
print 'The database is not empty. Let\'s import only diff files.'
|
|
||||||
|
|
||||||
# Finally launch the listening process.
|
def count_table(self, name):
|
||||||
while True:
|
"""Check if there is a table starting with name."""
|
||||||
import_queue = sorted(listdir(default['IMPORT_QUEUE']))
|
sql = 'select count(*) ' \
|
||||||
if len(import_queue) > 0:
|
'from information_schema.tables ' \
|
||||||
for diff in import_queue:
|
'where table_name like \'%s\';' % name
|
||||||
print 'Importing diff %s' % diff
|
# noinspection PyUnboundLocalVariable
|
||||||
command = ['imposm3', 'diff']
|
self.cursor.execute(sql)
|
||||||
command += ['-cachedir', default['CACHE']]
|
return self.cursor.fetchone()[0]
|
||||||
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
|
||||||
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
|
||||||
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
|
||||||
command += ['-srid', default['SRID']]
|
|
||||||
command += ['-diffdir', default['SETTINGS']]
|
|
||||||
command += ['-mapping', mapping_file]
|
|
||||||
command += ['-connection', postgis_uri]
|
|
||||||
command += [join(default['IMPORT_QUEUE'], diff)]
|
|
||||||
|
|
||||||
print ' '.join(command)
|
def run(self):
|
||||||
if call(command) == 0:
|
osm_tables = self.count_table('osm_%')
|
||||||
move(
|
if osm_tables < 1:
|
||||||
join(default['IMPORT_QUEUE'], diff),
|
# It means that the DB is empty. Let's import the PBF file.
|
||||||
join(default['IMPORT_DONE'], diff))
|
self._first_pbf_import()
|
||||||
|
else:
|
||||||
|
self.info('The database is not empty. Let\'s import only diff files.')
|
||||||
|
|
||||||
# Update the timestamp in the file.
|
self._import_diff()
|
||||||
database_timestamp = diff.split('.')[0].split('->-')[1]
|
|
||||||
file_path = join(default['SETTINGS'], 'timestamp.txt')
|
|
||||||
timestamp_file = open(file_path, 'w')
|
|
||||||
timestamp_file.write('%s\n' % database_timestamp)
|
|
||||||
timestamp_file.close()
|
|
||||||
|
|
||||||
print 'Import diff successful : %s' % diff
|
def _first_pbf_import(self):
|
||||||
else:
|
command = ['imposm3', 'import', '-diff', '-deployproduction']
|
||||||
print >> stderr, 'An error occured in imposm with a diff.'
|
command += ['-overwritecache', '-cachedir', self.default['CACHE']]
|
||||||
exit()
|
command += ['-srid', self.default['SRID']]
|
||||||
|
command += ['-dbschema-production',
|
||||||
|
self.default['DBSCHEMA_PRODUCTION']]
|
||||||
|
command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
|
||||||
|
command += ['-dbschema-backup', self.default['DBSCHEMA_BACKUP']]
|
||||||
|
command += ['-diffdir', self.default['SETTINGS']]
|
||||||
|
command += ['-mapping', self.mapping_file]
|
||||||
|
command += ['-read', self.osm_file]
|
||||||
|
command += ['-write', '-connection', self.postgis_uri]
|
||||||
|
self.info('The database is empty. Let\'s import the PBF : %s' % self.osm_file)
|
||||||
|
self.info(' '.join(command))
|
||||||
|
if not call(command) == 0:
|
||||||
|
msg = 'An error occured in imposm with the original file.'
|
||||||
|
self.error(msg)
|
||||||
|
else:
|
||||||
|
self.info('Import PBF successful : %s' % self.osm_file)
|
||||||
|
|
||||||
if len(listdir(default['IMPORT_QUEUE'])) == 0:
|
if self.post_import_file or self.qgis_style:
|
||||||
print 'Sleeping for %s seconds.' % default['TIME']
|
# Set the password for psql
|
||||||
sleep(float(default['TIME']))
|
environ['PGPASSWORD'] = self.default['PASSWORD']
|
||||||
|
|
||||||
|
if self.post_import_file:
|
||||||
|
self.import_custom_sql()
|
||||||
|
|
||||||
|
if self.qgis_style:
|
||||||
|
self.import_qgis_styles()
|
||||||
|
|
||||||
|
def _import_diff(self):
|
||||||
|
# Finally launch the listening process.
|
||||||
|
while True:
|
||||||
|
import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
|
||||||
|
if len(import_queue) > 0:
|
||||||
|
for diff in import_queue:
|
||||||
|
self.info('Importing diff %s' % diff)
|
||||||
|
command = ['imposm3', 'diff']
|
||||||
|
command += ['-cachedir', self.default['CACHE']]
|
||||||
|
command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']]
|
||||||
|
command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
|
||||||
|
command += ['-dbschema-backup', self.default['DBSCHEMA_BACKUP']]
|
||||||
|
command += ['-srid', self.default['SRID']]
|
||||||
|
command += ['-diffdir', self.default['SETTINGS']]
|
||||||
|
command += ['-mapping', self.mapping_file]
|
||||||
|
command += ['-connection', self.postgis_uri]
|
||||||
|
command += [join(self.default['IMPORT_QUEUE'], diff)]
|
||||||
|
|
||||||
|
self.info(' '.join(command))
|
||||||
|
if call(command) == 0:
|
||||||
|
move(
|
||||||
|
join(self.default['IMPORT_QUEUE'], diff),
|
||||||
|
join(self.default['IMPORT_DONE'], diff))
|
||||||
|
|
||||||
|
# Update the timestamp in the file.
|
||||||
|
database_timestamp = diff.split('.')[0].split('->-')[1]
|
||||||
|
self.update_timestamp(database_timestamp)
|
||||||
|
|
||||||
|
self.info('Import diff successful : %s' % diff)
|
||||||
|
else:
|
||||||
|
msg = 'An error occured in imposm with a diff.'
|
||||||
|
self.error(msg)
|
||||||
|
|
||||||
|
if len(listdir(self.default['IMPORT_QUEUE'])) == 0:
|
||||||
|
self.info('Sleeping for %s seconds.' % self.default['TIME'])
|
||||||
|
sleep(float(self.default['TIME']))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
importer = Importer()
|
||||||
|
importer.overwrite_environment()
|
||||||
|
importer.check_settings()
|
||||||
|
importer.create_timestamp()
|
||||||
|
importer.check_postgis()
|
||||||
|
importer.run()
|
||||||
|
|
|
||||||
Ładowanie…
Reference in New Issue