mirror
/
docker-osm
kopia lustrzana https://github.com/kartoza/docker-osm
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Merge pull request #24 from Gustry/improvments 

Improvments
pull/26/head
Etienne Trimaille 2015-12-23 11:28:57 +01:00
rodzic 19ac1596d2 3226af176b
commit cbbfe25469
11 zmienionych plików z 13663 dodań i 169 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
.gitignore vendored
Wyświetl plik

@ -1,7 +1,11 @@
.idea
*.*~
cache
import_done
import_queue
settings/*.pbf
settings/*.state.txt
settings/*.poly
settings/last.state.txt
settings/custom_url_diff.txt
settings/timestamp.txt

51
Makefile 100644
Wyświetl plik

@ -0,0 +1,51 @@
PROJECT_ID := dockerosm
build:
@echo
@echo "------------------------------------------------------------------"
@echo "Building in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -p $(PROJECT_ID) build
run:
@echo
@echo "------------------------------------------------------------------"
@echo "Running in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -p $(PROJECT_ID) up -d --no-recreate
rundev:
@echo
@echo "------------------------------------------------------------------"
@echo "Running in DEVELOPMENT mode"
@echo "------------------------------------------------------------------"
@docker-compose -p $(PROJECT_ID) up
kill:
@echo
@echo "------------------------------------------------------------------"
@echo "Killing in production mode"
@echo "------------------------------------------------------------------"
@docker-compose -p $(PROJECT_ID) kill
rm: kill
@echo
@echo "------------------------------------------------------------------"
@echo "Removing production instance!!! "
@echo "------------------------------------------------------------------"
@docker-compose -p $(PROJECT_ID) rm
ipdb:
@echo
@echo "------------------------------------------------------------------"
@echo "Database's IP"
@echo "------------------------------------------------------------------"
@docker inspect $(PROJECT_ID)_db_1 | grep '"IPAddress"' | head -1 | cut -d '"' -f 4
timestamp:
@echo
@echo "------------------------------------------------------------------"
@echo "Timestamp"
@echo "------------------------------------------------------------------"
@docker exec -t -i $(PROJECT_ID)_imposm_1 cat /home/settings/timestamp.txt

39
docker-compose.yml
Wyświetl plik

@ -12,6 +12,7 @@ storage:
#- ./cache:/home/cache
db:
# About the postgresql version, it should match in the dockerfile of docker-imposm3
image: kartoza/postgis:9.4-2.1
hostname: db
environment:
@ -30,27 +31,29 @@ imposm:
- PASSWORD=docker
- PORT=5432
- HOST=db
- DATEBASE=gis
- DATABASE=gis
# seconds between 2 executions of the script
- TIME=120
# folder for settings (with *.json and *.sql)
- SETTINGS = settings
- SETTINGS=settings
# folder for caching
- CACHE = cache
- CACHE=cache
# folder for diff which has been imported
- IMPORT_DONE = import_done
- IMPORT_DONE=import_done
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE = import_queue
- IMPORT_QUEUE=import_queue
# it can be 3857
- SRID = 4326
- SRID=4326
# see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
- OPTIMIZE = false
- OPTIMIZE=false
# see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_PRODUCTION = public
- DBSCHEMA_PRODUCTION=public
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_IMPORT = import
- DBSCHEMA_IMPORT=import
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_BACKUP = backup
- DBSCHEMA_BACKUP=backup
# Install some styles if you are using the default mapping. It can be 'yes' or 'no'
- QGIS_STYLE=yes
osmupdate:
@ -61,18 +64,18 @@ osmupdate:
# These are all currently the defaults but listed here for your
# convenience if you want to change them
# the maximum time range to assamble a cumulated changefile.
- MAX_DAYS = 100
- MAX_DAYS=100
# osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
- DIFF = sporadic
- DIFF=sporadic
# argument to determine the maximum number of parallely processed changefiles.
- MAX_MERGE = 7
- MAX_MERGE=7
# define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
- COMPRESSION_LEVEL = 1
- COMPRESSION_LEVEL=1
# change the URL to use a custom URL to fetch regional file updates.
- BASE_URL = http://planet.openstreetmap.org/replication/
- BASE_URL=http://planet.openstreetmap.org/replication/
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE = import_queue
- IMPORT_QUEUE=import_queue
# folder for diff which has been imported
- IMPORT_DONE = import_done
- IMPORT_DONE=import_done
# seconds between 2 executions of the script
- TIME = 120
- TIME=120

6
docker-imposm3/Dockerfile
Wyświetl plik

@ -1,9 +1,9 @@
FROM golang:1.4
FROM golang:1.5
MAINTAINER Etienne Trimaille <etienne@kartoza.com>
RUN apt-get update && apt-get install -y python-pip \
libprotobuf-dev libleveldb-dev libgeos-dev \
libpq-dev python-dev \
libpq-dev python-dev postgresql-client-9.4 \
--no-install-recommends
RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
@ -11,7 +11,7 @@ RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
WORKDIR $GOPATH
RUN go get github.com/tools/godep
RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/imposm3
RUN cd src/github.com/omniscale/imposm3 && godep go install ./...
RUN cd src/github.com/omniscale/imposm3 && godep go install ./
ADD requirements.txt /home/requirements.txt
RUN pip install -r /home/requirements.txt

65
docker-imposm3/importer.py
Wyświetl plik

@ -44,17 +44,23 @@ default = {
'OPTIMIZE': 'false',
'DBSCHEMA_PRODUCTION': 'public',
'DBSCHEMA_IMPORT': 'import',
'DBSCHEMA_BACKUP': 'backup'
'DBSCHEMA_BACKUP': 'backup',
'QGIS_STYLE': 'yes'
}
# Check if we overwrite default values.
for key in default.keys():
if key in environ:
for key in environ.keys():
if key in default.keys():
default[key] = environ[key]
# Check valid SRID.
if default['SRID'] not in ['4326', '3857']:
print >> stderr, 'SRID not supported : %s' % default['srid']
print >> stderr, 'SRID not supported : %s' % default['SRID']
exit()
# Check valid QGIS_STYLE.
if default['QGIS_STYLE'] not in ['yes', 'no']:
print >> stderr, 'QGIS_STYLE not supported : %s' % default['QGIS_STYLE']
exit()
# Check folders.
@ -70,13 +76,11 @@ for folder in folders:
exit()
# Test files
state_file = None
osm_file = None
mapping_file = None
post_import_file = None
qgis_style = None
for f in listdir(default['SETTINGS']):
if f == 'last.state.txt':
state_file = join(default['SETTINGS'], f)
if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f)
@ -84,17 +88,16 @@ for f in listdir(default['SETTINGS']):
if f.endswith('.json'):
mapping_file = join(default['SETTINGS'], f)
if f.endswith('.sql'):
if f == 'post-pbf-import.sql':
post_import_file = join(default['SETTINGS'], f)
if f == 'qgis_style.sql':
qgis_style = join(default['SETTINGS'], f)
if not osm_file:
print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
exit()
if not state_file:
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
exit()
if not mapping_file:
print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
exit()
@ -104,6 +107,14 @@ if not post_import_file:
else:
print '%s detected for post import.' % post_import_file
if not qgis_style and default['QGIS_STYLE'] == 'yes':
print >> stderr, 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % default['SETTINGS']
exit()
elif qgis_style and default['QGIS_STYLE']:
print '%s detected for QGIS styling.' % qgis_style
else:
print 'Not using QGIS default styles.'
# In docker-compose, we should wait for the DB is ready.
print 'The checkup is OK. The container will continue soon, after the database.'
sleep(45)
@ -157,9 +168,27 @@ if osm_tables < 1:
else:
print 'Import PBF successful : %s' % osm_file
if post_import_file or qgis_style:
# Set the password for psql
environ['PGPASSWORD'] = default['PASSWORD']
if post_import_file:
for sql in open(post_import_file):
cursor.execute(sql)
print 'Running the post import SQL file.'
command = ['psql']
command += ['-h', default['HOST']]
command += ['-U', default['USER']]
command += ['-d', default['DATABASE']]
command += ['-f', post_import_file]
call(command)
if qgis_style:
'Installing QGIS styles.'
command = ['psql']
command += ['-h', default['HOST']]
command += ['-U', default['USER']]
command += ['-d', default['DATABASE']]
command += ['-f', qgis_style]
call(command)
else:
print 'The database is not empty. Let\'s import only diff files.'
@ -185,6 +214,14 @@ while True:
move(
join(default['IMPORT_QUEUE'], diff),
join(default['IMPORT_DONE'], diff))
# Update the timestamp in the file.
database_timestamp = diff.split('.')[0].split('->-')[1]
file_path = join(default['SETTINGS'], 'timestamp.txt')
timestamp_file = open(file_path, 'w')
timestamp_file.write('%s\n' % database_timestamp)
timestamp_file.close()
print 'Import diff successful : %s' % diff
else:
print >> stderr, 'An error occured in imposm with a diff.'

2
docker-osmupdate/Dockerfile
Wyświetl plik

@ -21,7 +21,9 @@ RUN apt-get -y install ca-certificates rpl pwgen
RUN apt-get -y install osmctools wget gzip gcc libc-dev zlib1g-dev
WORKDIR /home
ADD osmupdate.c /home/osmupdate.c
ADD osmconvert.c /home/osmconvert.c
RUN gcc -x c - -o osmupdate osmupdate.c
RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz
# Add the python script which will call osmupdate
ADD download.py /home/download.py

57
docker-osmupdate/download.py
Wyświetl plik

@ -22,15 +22,11 @@
from os.path import exists, join, isabs, abspath
from os import listdir, environ
from sys import exit
from subprocess import call
from subprocess import call, Popen, PIPE
from datetime import datetime
from time import sleep
from sys import stderr
# In docker-compose, we should wait for the DB is ready.
print 'The container will start soon, after the database.'
sleep(45)
# Default values which can be overwritten.
default = {
'MAX_DAYS': '100',
@ -44,8 +40,8 @@ default = {
'TIME': 120,
}
for key in default.keys():
if key in environ:
for key in environ.keys():
if key in default.keys():
default[key] = environ[key]
# Folders
@ -61,19 +57,12 @@ for folder in folders:
exit()
# Test files
state_file = None
osm_file = None
poly_file = None
for f in listdir(default['SETTINGS']):
if f == 'last.state.txt':
state_file = join(default['SETTINGS'], f)
if f.endswith('.pbf'):
osm_file = join(default['SETTINGS'], f)
if f.endswith('.poly'):
poly_file = join(default['SETTINGS'], f)
"""
# Todo : need fix custom URL and sporadic diff : daily, hourly and minutely
if f == 'custom_url_diff.txt':
@ -81,44 +70,40 @@ for f in listdir(default['SETTINGS']):
default['BASE_URL'] = content_file.read()
"""
if not state_file:
print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
exit()
if not osm_file:
print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS']
exit()
if not poly_file:
print 'No *.poly detected in %s' % default['SETTINGS']
else:
print '%s detected for clipping.' % poly_file
# In docker-compose, we should wait for the DB is ready.
print 'The checkup is OK. The container will continue soon, after the database.'
sleep(45)
# Finally launch the listening process.
while True:
# Check if diff to be imported is empty. If not, take the latest diff.
diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
if len(diff_to_be_imported):
timestamp = diff_to_be_imported[-1].split('.')[0]
print "Timestamp from the latest not imported diff : %s" % timestamp
file_name = diff_to_be_imported[-1].split('.')[0]
timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest not imported diff : %s' % timestamp
else:
# Check if imported diff is empty. If not, take the latest diff.
imported_diff = sorted(listdir(default['IMPORT_DONE']))
if len(imported_diff):
print "Timestamp from the latest imported diff : %s" % timestamp
timestamp = imported_diff[-1].split('.')[0]
file_name = imported_diff[-1].split('.')[0]
timestamp = file_name.split('->-')[1]
print 'Timestamp from the latest imported diff : %s' % timestamp
else:
# Take the timestamp from original file.
state_file_settings = {}
with open(state_file) as a_file:
for line in a_file:
if '=' in line:
name, value = line.partition("=")[::2]
state_file_settings[name] = value
command = ['osmconvert', osm_file, '--out-timestamp']
processus = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
timestamp, err = processus.communicate()
timestamp = state_file_settings['timestamp'].strip()
print "Timestamp from the original state file : %s" % timestamp
# Remove new line
timestamp = timestamp.strip()
print 'Timestamp from the original state file : %s' % timestamp
# Removing some \ in the timestamp.
timestamp = timestamp.replace('\\', '')
@ -129,13 +114,11 @@ while True:
print 'Current time : %s' % current_time
# Destination
file_name = '%s.osc.gz' % current_time
file_name = '%s->-%s.osc.gz' % (timestamp, current_time)
file_path = join(default['IMPORT_QUEUE'], file_name)
# Command
command = ['osmupdate', '-v']
if poly_file:
command.append('-B=%s' % poly_file)
command += ['--max-days=' + default['MAX_DAYS']]
command += [default['DIFF']]
command += ['--max-merge=' + default['MAX_MERGE']]

11941
docker-osmupdate/osmconvert.c 100644
Wyświetl plik

Plik diff jest za duży Load Diff

247
settings/mapping.json
Wyświetl plik

@ -1,4 +1,46 @@
{
"generalized_tables": {
"waterareas_gen1": {
"source": "waterareas",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"waterareas_gen0": {
"source": "waterareas_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
},
"roads_gen0": {
"source": "roads_gen1",
"sql_filter": null,
"tolerance": 200.0
},
"roads_gen1": {
"source": "roads",
"sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')",
"tolerance": 50.0
},
"waterways_gen0": {
"source": "waterways_gen1",
"sql_filter": null,
"tolerance": 200
},
"waterways_gen1": {
"source": "waterways",
"sql_filter": null,
"tolerance": 50.0
},
"landusages_gen1": {
"source": "landusages",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"landusages_gen0": {
"source": "landusages_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
}
},
"tables": {
"landusages": {
"fields": [
@ -29,58 +71,57 @@
},
{
"args": {
"ranks": [
"pedestrian",
"footway",
"playground",
"park",
"forest",
"cemetery",
"farmyard",
"farm",
"farmland",
"wood",
"meadow",
"grass",
"wetland",
"village_green",
"recreation_ground",
"garden",
"sports_centre",
"pitch",
"common",
"allotments",
"golf_course",
"university",
"school",
"college",
"library",
"baracks",
"fuel",
"parking",
"nature_reserve",
"cinema",
"theatre",
"place_of_worship",
"hospital",
"scrub",
"orchard",
"vineyard",
"zoo",
"quarry",
"residential",
"retail",
"commercial",
"industrial",
"railway",
"heath",
"values": [
"land",
"island",
"land"
"heath",
"railway",
"industrial",
"commercial",
"retail",
"residential",
"quarry",
"zoo",
"vineyard",
"orchard",
"scrub",
"hospital",
"place_of_worship",
"theatre",
"cinema",
"nature_reserve",
"parking",
"fuel",
"baracks",
"library",
"college",
"school",
"university",
"golf_course",
"allotments",
"common",
"pitch",
"sports_centre",
"garden",
"recreation_ground",
"village_green",
"wetland",
"grass",
"meadow",
"wood",
"farmland",
"farm",
"farmyard",
"cemetery",
"forest",
"park",
"playground",
"footway",
"pedestrian"
]
},
"type": "zorder",
"name": "z_order",
"key": "z_order"
"type": "enumerate",
"name": "z_order"
}
],
"type": "polygon",
@ -116,7 +157,6 @@
],
"natural": [
"wood",
"water",
"land",
"scrub",
"wetland",
@ -160,9 +200,6 @@
"highway": [
"pedestrian",
"footway"
],
"waterway": [
"riverbank"
]
}
},
@ -196,37 +233,6 @@
]
}
},
"amenity_areas": {
"_comment": "for testing duplicate inserts with __any__ and exact match",
"fields": [
{
"type": "id",
"name": "osm_id",
"key": null
},
{
"type": "geometry",
"name": "geometry",
"key": null
},
{
"type": "string",
"name": "name",
"key": "name"
},
{
"type": "mapping_value",
"name": "type",
"key": null
}
],
"type": "polygon",
"mapping": {
"building": [
"shop"
]
}
},
"places": {
"fields": [
{
@ -251,22 +257,21 @@
},
{
"args": {
"ranks": [
"country",
"state",
"region",
"county",
"city",
"town",
"village",
"hamlet",
"values": [
"locality",
"suburb",
"locality"
"hamlet",
"village",
"town",
"city",
"county",
"region",
"state",
"country"
]
},
"type": "zorder",
"name": "z_order",
"key": "z_order"
"type": "enumerate",
"name": "z_order"
},
{
"type": "integer",
@ -819,6 +824,54 @@
"__any__"
]
}
},
"waterareas": {
"fields": [
{
"type": "id",
"name": "osm_id",
"key": null
},
{
"type": "geometry",
"name": "geometry",
"key": null
},
{
"type": "string",
"name": "name",
"key": "name"
},
{
"type": "mapping_value",
"name": "type",
"key": null
},
{
"type": "pseudoarea",
"name": "area",
"key": null
}
],
"type": "polygon",
"mapping": {
"waterway": [
"riverbank"
],
"landuse": [
"basin",
"reservoir"
],
"natural": [
"water"
],
"amenity": [
"swimming_pool"
],
"leisure": [
"swimming_pool"
]
}
}
}
}

1419
settings/qgis_style.sql 100755
Wyświetl plik

Plik diff jest za duży Load Diff

1
settings/timestamp.txt 100644
Wyświetl plik

@ -0,0 +1 @@
UNDEFINED