kopia lustrzana https://github.com/kartoza/docker-osm
cleaning and improve log
Etienne Trimaille
rodzic
139d20962c
commit
c413e18fb7
|
|
@ -7,6 +7,9 @@ storage:
|
|||
- /home/import_done
|
||||
- /home/import_queue
|
||||
- /home/cache
|
||||
#- ./import_done:/home/import_done
|
||||
#- ./import_queue:/home/import_queue
|
||||
#- ./cache:/home/cache
|
||||
|
||||
db:
|
||||
image: kartoza/postgis:9.4-2.1
|
||||
|
|
@ -34,8 +37,6 @@ imposm:
|
|||
- SETTINGS = settings
|
||||
# folder for caching
|
||||
- CACHE = cache
|
||||
# folder the OSM file
|
||||
- BASE_PBF = base_pbf
|
||||
# folder for diff which has been imported
|
||||
- IMPORT_DONE = import_done
|
||||
# folder for diff which hasn't been imported yet
|
||||
|
|
@ -53,7 +54,6 @@ imposm:
|
|||
|
||||
|
||||
osmupdate:
|
||||
#image: etrimaille/osmupdate
|
||||
build: docker-osmupdate
|
||||
volumes_from:
|
||||
- storage
|
||||
|
|
@ -70,7 +70,9 @@ osmupdate:
|
|||
- COMPRESSION_LEVEL = 1
|
||||
# change the URL to use a custom URL to fetch regional file updates.
|
||||
- BASE_URL = http://planet.openstreetmap.org/replication/
|
||||
# folder for diff which hasn't been imported yet
|
||||
- IMPORT_QUEUE = import_queue
|
||||
# folder for diff which has been imported
|
||||
- IMPORT_DONE = import_done
|
||||
- OSM_PBF = osm_pbf
|
||||
- TIME = 120, seconds between each execution of the script
|
||||
# seconds between 2 executions of the script
|
||||
- TIME = 120
|
||||
|
|
|
|||
|
|
@ -28,9 +28,6 @@ from subprocess import call
|
|||
from time import sleep
|
||||
from sys import stderr
|
||||
|
||||
# In docker-compose, we should wait for the DB is ready.
|
||||
sleep(45)
|
||||
|
||||
# All these default values can be overwritten by env vars
|
||||
default = {
|
||||
'TIME': 120,
|
||||
|
|
@ -60,6 +57,57 @@ if default['SRID'] not in ['4326', '3857']:
|
|||
print >> stderr, 'SRID not supported : %s' % default['srid']
|
||||
exit()
|
||||
|
||||
# Check folders.
|
||||
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
|
||||
for folder in folders:
|
||||
if not isabs(default[folder]):
|
||||
# Get the absolute path.
|
||||
default[folder] = abspath(default[folder])
|
||||
|
||||
# Test the folder
|
||||
if not exists(default[folder]):
|
||||
print >> stderr, 'The folder %s does not exist.' % default[folder]
|
||||
exit()
|
||||
|
||||
# Test files
|
||||
state_file = None
|
||||
osm_file = None
|
||||
mapping_file = None
|
||||
post_import_file = None
|
||||
for f in listdir(default['SETTINGS']):
|
||||
if f == 'last.state.txt':
|
||||
state_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.pbf'):
|
||||
osm_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.json'):
|
||||
mapping_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.sql'):
|
||||
post_import_file = join(default['SETTINGS'], f)
|
||||
|
||||
if not osm_file:
|
||||
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not state_file:
|
||||
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not mapping_file:
|
||||
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not post_import_file:
|
||||
print 'No *.sql detected in %s' % default['SETTINGS']
|
||||
else:
|
||||
print '%s detected for post import.' % post_import_file
|
||||
|
||||
# In docker-compose, we should wait for the DB is ready.
|
||||
print 'The checkup is OK. The container will continue soon, after the database.'
|
||||
sleep(45)
|
||||
|
||||
# Check postgis.
|
||||
try:
|
||||
connection = connect(
|
||||
|
|
@ -79,52 +127,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % (
|
|||
default['HOST'],
|
||||
default['DATABASE'])
|
||||
|
||||
# Check folders.
|
||||
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
|
||||
for folder in folders:
|
||||
if not isabs(default[folder]):
|
||||
# Get the absolute path.
|
||||
default[folder] = abspath(default[folder])
|
||||
|
||||
# Test the folder
|
||||
if not exists(default[folder]):
|
||||
print >> stderr, 'The folder %s does not exist.' % default[folder]
|
||||
exit()
|
||||
|
||||
# Test files
|
||||
state_file = None
|
||||
osm_file = None
|
||||
mapping_file = None
|
||||
post_import_file = None
|
||||
for f in listdir(default['SETTINGS']):
|
||||
if f.endswith('.state.txt'):
|
||||
state_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.pbf'):
|
||||
osm_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.json'):
|
||||
mapping_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.sql'):
|
||||
post_import_file = join(default['SETTINGS'], f)
|
||||
|
||||
if not osm_file:
|
||||
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not state_file:
|
||||
print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not mapping_file:
|
||||
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not post_import_file:
|
||||
print 'No *.sql detected in %s' % default['SETTINGS']
|
||||
else:
|
||||
print '%s detected for post import.' % post_import_file
|
||||
|
||||
# Check if there is a table starting with 'osm_'
|
||||
sql = 'select count(*) ' \
|
||||
|
|
@ -134,25 +137,31 @@ sql = 'select count(*) ' \
|
|||
cursor.execute(sql)
|
||||
osm_tables = cursor.fetchone()[0]
|
||||
if osm_tables < 1:
|
||||
# It means that the DB is empty. Let's import the file.
|
||||
commands = ['imposm3', 'import', '-diff', '-deployproduction']
|
||||
commands += ['-overwritecache', '-cachedir', default['CACHE']]
|
||||
commands += ['-srid', default['SRID']]
|
||||
commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
||||
commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
||||
commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
||||
commands += ['-diffdir', default['SETTINGS']]
|
||||
commands += ['-mapping', mapping_file]
|
||||
commands += ['-read', osm_file]
|
||||
commands += ['-write', '-connection', postgis_uri]
|
||||
# It means that the DB is empty. Let's import the PBF file.
|
||||
command = ['imposm3', 'import', '-diff', '-deployproduction']
|
||||
command += ['-overwritecache', '-cachedir', default['CACHE']]
|
||||
command += ['-srid', default['SRID']]
|
||||
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
||||
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
||||
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
||||
command += ['-diffdir', default['SETTINGS']]
|
||||
command += ['-mapping', mapping_file]
|
||||
command += ['-read', osm_file]
|
||||
command += ['-write', '-connection', postgis_uri]
|
||||
|
||||
if not call(commands) == 0:
|
||||
print 'The database is empty. Let\'s import the PBF : %s' % osm_file
|
||||
print ' '.join(command)
|
||||
if not call(command) == 0:
|
||||
print >> stderr, 'An error occured in imposm with the original file.'
|
||||
exit()
|
||||
else:
|
||||
print 'Import PBF successful : %s' % osm_file
|
||||
|
||||
if post_import_file:
|
||||
for sql in open(post_import_file):
|
||||
cursor.execute(sql)
|
||||
else:
|
||||
print 'The database is not empty. Let\'s import only diff files.'
|
||||
|
||||
# Finally launch the listening process.
|
||||
while True:
|
||||
|
|
@ -160,21 +169,23 @@ while True:
|
|||
if len(import_queue) > 0:
|
||||
for diff in import_queue:
|
||||
print 'Importing diff %s' % diff
|
||||
commands = ['imposm3', 'diff']
|
||||
commands += ['-cachedir', default['CACHE']]
|
||||
commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
||||
commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
||||
commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
||||
commands += ['-srid', default['SRID']]
|
||||
commands += ['-diffdir', default['SETTINGS']]
|
||||
commands += ['-mapping', mapping_file]
|
||||
commands += ['-connection', postgis_uri]
|
||||
commands += [join(default['IMPORT_QUEUE'], diff)]
|
||||
command = ['imposm3', 'diff']
|
||||
command += ['-cachedir', default['CACHE']]
|
||||
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
|
||||
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
|
||||
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
|
||||
command += ['-srid', default['SRID']]
|
||||
command += ['-diffdir', default['SETTINGS']]
|
||||
command += ['-mapping', mapping_file]
|
||||
command += ['-connection', postgis_uri]
|
||||
command += [join(default['IMPORT_QUEUE'], diff)]
|
||||
|
||||
if call(commands) == 0:
|
||||
print ' '.join(command)
|
||||
if call(command) == 0:
|
||||
move(
|
||||
join(default['IMPORT_QUEUE'], diff),
|
||||
join(default['IMPORT_DONE'], diff))
|
||||
print 'Import diff successful : %s' % diff
|
||||
else:
|
||||
print >> stderr, 'An error occured in imposm with a diff.'
|
||||
exit()
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
***************************************************************************/
|
||||
"""
|
||||
|
||||
from os.path import exists, join, isabs, abspath, isfile
|
||||
from os.path import exists, join, isabs, abspath
|
||||
from os import listdir, environ
|
||||
from sys import exit
|
||||
from subprocess import call
|
||||
|
|
@ -28,6 +28,7 @@ from time import sleep
|
|||
from sys import stderr
|
||||
|
||||
# In docker-compose, we should wait for the DB is ready.
|
||||
print 'The container will start soon, after the database.'
|
||||
sleep(45)
|
||||
|
||||
# Default values which can be overwritten.
|
||||
|
|
@ -64,7 +65,7 @@ state_file = None
|
|||
osm_file = None
|
||||
poly_file = None
|
||||
for f in listdir(default['SETTINGS']):
|
||||
if f.endswith('.state.txt'):
|
||||
if f == 'last.state.txt':
|
||||
state_file = join(default['SETTINGS'], f)
|
||||
|
||||
if f.endswith('.pbf'):
|
||||
|
|
@ -81,7 +82,7 @@ for f in listdir(default['SETTINGS']):
|
|||
"""
|
||||
|
||||
if not state_file:
|
||||
print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
|
||||
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
|
||||
exit()
|
||||
|
||||
if not osm_file:
|
||||
|
|
@ -93,22 +94,22 @@ if not poly_file:
|
|||
else:
|
||||
print '%s detected for clipping.' % poly_file
|
||||
|
||||
# Finally launch the listening process.
|
||||
while True:
|
||||
# Check if diff to be imported is empty. If not, take the latest diff.
|
||||
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
|
||||
if len(diff_to_be_imported):
|
||||
print "Timestamp from the lastest not imported diff."
|
||||
timestamp = diff_to_be_imported[-1].split('.')[0]
|
||||
print "Timestamp from the latest not imported diff : %s" % timestamp
|
||||
else:
|
||||
# Check if imported diff is empty. If not, take the latest diff.
|
||||
imported_diff = sorted(listdir(default['IMPORT_DONE']))
|
||||
if len(imported_diff):
|
||||
print "Timestamp from the lastest imported diff."
|
||||
print "Timestamp from the latest imported diff : %s" % timestamp
|
||||
timestamp = imported_diff[-1].split('.')[0]
|
||||
|
||||
else:
|
||||
# Take the timestamp from original file.
|
||||
print "Timestamp from the original state file."
|
||||
state_file_settings = {}
|
||||
with open(state_file) as a_file:
|
||||
for line in a_file:
|
||||
|
|
@ -117,6 +118,7 @@ while True:
|
|||
state_file_settings[name] = value
|
||||
|
||||
timestamp = state_file_settings['timestamp'].strip()
|
||||
print "Timestamp from the original state file : %s" % timestamp
|
||||
|
||||
# Removing some \ in the timestamp.
|
||||
timestamp = timestamp.replace('\\', '')
|
||||
|
|
@ -142,8 +144,9 @@ while True:
|
|||
command.append(timestamp)
|
||||
command.append(file_path)
|
||||
|
||||
print ' '.join(command)
|
||||
if call(command) != 0:
|
||||
print >> stderr, 'An error occured in osmupdate.'
|
||||
print >> stderr, 'An error occured in osmupdate. Let\'s try again.'
|
||||
# Sleep less.
|
||||
print 'Sleeping for 2 seconds.'
|
||||
sleep(2.0)
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue