mirror
/
docker-osm
kopia lustrzana https://github.com/kartoza/docker-osm
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Merge pull request #16 from Gustry/port 

Merge cleaning and improve logs, ports, pg version
pull/26/head
Etienne Trimaille 2015-12-18 15:10:44 +01:00
rodzic 08717310fe c413e18fb7
commit 19ac1596d2
3 zmienionych plików z 100 dodań i 87 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

17
docker-compose.yml
Wyświetl plik

@ -7,16 +7,16 @@ storage:
- /home/import_done
- /home/import_queue
- /home/cache
#- ./import_done:/home/import_done
#- ./import_queue:/home/import_queue
#- ./cache:/home/cache
db:
image: kartoza/postgis
image: kartoza/postgis:9.4-2.1
hostname: db
environment:
- USERNAME=docker
- PASS=docker
# - HSTORE=true
ports:
- 5432:5432
imposm:
#image: kartoza/imposm
@ -37,8 +37,6 @@ imposm:
- SETTINGS = settings
# folder for caching
- CACHE = cache
# folder the OSM file
- BASE_PBF = base_pbf
# folder for diff which has been imported
- IMPORT_DONE = import_done
# folder for diff which hasn't been imported yet
@ -56,7 +54,6 @@ imposm:
osmupdate:
#image: etrimaille/osmupdate
build: docker-osmupdate
volumes_from:
- storage
@ -73,7 +70,9 @@ osmupdate:
- COMPRESSION_LEVEL = 1
# change the URL to use a custom URL to fetch regional file updates.
- BASE_URL = http://planet.openstreetmap.org/replication/
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE = import_queue
# folder for diff which has been imported
- IMPORT_DONE = import_done
- OSM_PBF = osm_pbf
- TIME = 120, seconds between each execution of the script
# seconds between 2 executions of the script
- TIME = 120

153
docker-imposm3/importer.py
Wyświetl plik

@ -28,9 +28,6 @@ from subprocess import call
from time import sleep
from sys import stderr
# In docker-compose, we should wait for the DB is ready.
sleep(45)
# All these default values can be overwritten by env vars
default = {
'TIME': 120,
@ -60,6 +57,57 @@ if default['SRID'] not in ['4326', '3857']:
print >> stderr, 'SRID not supported : %s' % default['srid']
exit()
# Check folders.
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
for folder in folders:
if not isabs(default[folder]):
# Get the absolute path.
default[folder] = abspath(default[folder])
# Test the folder
if not exists(default[folder]):
print >> stderr, 'The folder %s does not exist.' % default[folder]
exit()
# Test files
state_file = None
osm_file = None
mapping_file = None
post_import_file = None
for f in listdir(default['SETTINGS']):
if f == 'last.state.txt':
state_file = join(default['SETTINGS'], f)
if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f)
if f.endswith('.json'):
mapping_file = join(default['SETTINGS'], f)
if f.endswith('.sql'):
post_import_file = join(default['SETTINGS'], f)
if not osm_file:
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
exit()
if not state_file:
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
exit()
if not mapping_file:
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
exit()
if not post_import_file:
print 'No *.sql detected in %s' % default['SETTINGS']
else:
print '%s detected for post import.' % post_import_file
# In docker-compose, we should wait for the DB is ready.
print 'The checkup is OK. The container will continue soon, after the database.'
sleep(45)
# Check postgis.
try:
connection = connect(
@ -79,52 +127,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % (
default['HOST'],
default['DATABASE'])
# Check folders.
folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
for folder in folders:
if not isabs(default[folder]):
# Get the absolute path.
default[folder] = abspath(default[folder])
# Test the folder
if not exists(default[folder]):
print >> stderr, 'The folder %s does not exist.' % default[folder]
exit()
# Test files
state_file = None
osm_file = None
mapping_file = None
post_import_file = None
for f in listdir(default['SETTINGS']):
if f.endswith('.state.txt'):
state_file = join(default['SETTINGS'], f)
if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f)
if f.endswith('.json'):
mapping_file = join(default['SETTINGS'], f)
if f.endswith('.sql'):
post_import_file = join(default['SETTINGS'], f)
if not osm_file:
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
exit()
if not state_file:
print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
exit()
if not mapping_file:
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
exit()
if not post_import_file:
print 'No *.sql detected in %s' % default['SETTINGS']
else:
print '%s detected for post import.' % post_import_file
# Check if there is a table starting with 'osm_'
sql = 'select count(*) ' \
@ -134,25 +137,31 @@ sql = 'select count(*) ' \
cursor.execute(sql)
osm_tables = cursor.fetchone()[0]
if osm_tables < 1:
# It means that the DB is empty. Let's import the file.
commands = ['imposm3', 'import', '-diff', '-deployproduction']
commands += ['-overwritecache', '-cachedir', default['CACHE']]
commands += ['-srid', default['SRID']]
commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
commands += ['-diffdir', default['SETTINGS']]
commands += ['-mapping', mapping_file]
commands += ['-read', osm_file]
commands += ['-write', '-connection', postgis_uri]
# It means that the DB is empty. Let's import the PBF file.
command = ['imposm3', 'import', '-diff', '-deployproduction']
command += ['-overwritecache', '-cachedir', default['CACHE']]
command += ['-srid', default['SRID']]
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
command += ['-diffdir', default['SETTINGS']]
command += ['-mapping', mapping_file]
command += ['-read', osm_file]
command += ['-write', '-connection', postgis_uri]
if not call(commands) == 0:
print 'The database is empty. Let\'s import the PBF : %s' % osm_file
print ' '.join(command)
if not call(command) == 0:
print >> stderr, 'An error occured in imposm with the original file.'
exit()
else:
print 'Import PBF successful : %s' % osm_file
if post_import_file:
for sql in open(post_import_file):
cursor.execute(sql)
else:
print 'The database is not empty. Let\'s import only diff files.'
# Finally launch the listening process.
while True:
@ -160,21 +169,23 @@ while True:
if len(import_queue) > 0:
for diff in import_queue:
print 'Importing diff %s' % diff
commands = ['imposm3', 'diff']
commands += ['-cachedir', default['CACHE']]
commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
commands += ['-srid', default['SRID']]
commands += ['-diffdir', default['SETTINGS']]
commands += ['-mapping', mapping_file]
commands += ['-connection', postgis_uri]
commands += [join(default['IMPORT_QUEUE'], diff)]
command = ['imposm3', 'diff']
command += ['-cachedir', default['CACHE']]
command += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', default['DBSCHEMA_IMPORT']]
command += ['-dbschema-backup', default['DBSCHEMA_BACKUP']]
command += ['-srid', default['SRID']]
command += ['-diffdir', default['SETTINGS']]
command += ['-mapping', mapping_file]
command += ['-connection', postgis_uri]
command += [join(default['IMPORT_QUEUE'], diff)]
if call(commands) == 0:
print ' '.join(command)
if call(command) == 0:
move(
join(default['IMPORT_QUEUE'], diff),
join(default['IMPORT_DONE'], diff))
print 'Import diff successful : %s' % diff
else:
print >> stderr, 'An error occured in imposm with a diff.'
exit()

17
docker-osmupdate/download.py
Wyświetl plik

@ -19,7 +19,7 @@
***************************************************************************/
"""
from os.path import exists, join, isabs, abspath, isfile
from os.path import exists, join, isabs, abspath
from os import listdir, environ
from sys import exit
from subprocess import call
@ -28,6 +28,7 @@ from time import sleep
from sys import stderr
# In docker-compose, we should wait for the DB is ready.
print 'The container will start soon, after the database.'
sleep(45)
# Default values which can be overwritten.
@ -64,7 +65,7 @@ state_file = None
osm_file = None
poly_file = None
for f in listdir(default['SETTINGS']):
if f.endswith('.state.txt'):
if f == 'last.state.txt':
state_file = join(default['SETTINGS'], f)
if f.endswith('.pbf'):
@ -81,7 +82,7 @@ for f in listdir(default['SETTINGS']):
"""
if not state_file:
print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
exit()
if not osm_file:
@ -93,22 +94,22 @@ if not poly_file:
else:
print '%s detected for clipping.' % poly_file
# Finally launch the listening process.
while True:
# Check if diff to be imported is empty. If not, take the latest diff.
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
if len(diff_to_be_imported):
print "Timestamp from the lastest not imported diff."
timestamp = diff_to_be_imported[-1].split('.')[0]
print "Timestamp from the latest not imported diff : %s" % timestamp
else:
# Check if imported diff is empty. If not, take the latest diff.
imported_diff = sorted(listdir(default['IMPORT_DONE']))
if len(imported_diff):
print "Timestamp from the lastest imported diff."
print "Timestamp from the latest imported diff : %s" % timestamp
timestamp = imported_diff[-1].split('.')[0]
else:
# Take the timestamp from original file.
print "Timestamp from the original state file."
state_file_settings = {}
with open(state_file) as a_file:
for line in a_file:
@ -117,6 +118,7 @@ while True:
state_file_settings[name] = value
timestamp = state_file_settings['timestamp'].strip()
print "Timestamp from the original state file : %s" % timestamp
# Removing some \ in the timestamp.
timestamp = timestamp.replace('\\', '')
@ -142,8 +144,9 @@ while True:
command.append(timestamp)
command.append(file_path)
print ' '.join(command)
if call(command) != 0:
print >> stderr, 'An error occured in osmupdate.'
print >> stderr, 'An error occured in osmupdate. Let\'s try again.'
# Sleep less.
print 'Sleeping for 2 seconds.'
sleep(2.0)