mirror
/
docker-osm
kopia lustrzana https://github.com/kartoza/docker-osm
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Fix for python path issues due to wrong version installed in docker-o… (#53) 

* Fix for python path issues due to wrong version installed in docker-osm-update

* Pythonize to version 3

* Fixes for PR based on comments from Gustry
pull/55/head
mazano 2018-06-14 12:19:06 +02:00 zatwierdzone przez Etienne Trimaille
rodzic 182ff0132a
commit b8ef57c49a
6 zmienionych plików z 128 dodań i 123 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

182
docker-compose.yml
Wyświetl plik

@ -1,96 +1,94 @@
storage: version: '2.1'
image: ubuntu:latest
container_name: dockerosm_storage
hostname: storage
volumes:
# These are sharable to other containers
- ./settings:/home/settings
- /home/import_done
- /home/import_queue
- /home/cache
#- ./import_done:/home/import_done
#- ./import_queue:/home/import_queue
#- ./cache:/home/cache
db: services:
# About the postgresql version, it should match in the dockerfile of docker-imposm3 storage:
image: kartoza/postgis:9.6-2.4 image: ubuntu:bionic
container_name: dockerosm_db volumes:
hostname: db # These are sharable to other containers
environment: - ./settings:/home/settings
- POSTGRES_USER=docker - /home/import_done
- POSTGRES_PASS=docker - /home/import_queue
- POSTGRES_DBNAME=gis - /home/cache
volumes_from:
- storage
# Uncomment to use the postgis database from outside the docker network
# ports:
# - "5432:5432"
imposm: db:
# image: etrimaille/docker-osm:imposm-latest # About the postgresql version, it should match in the dockerfile of docker-imposm3
build: docker-imposm3 image: kartoza/postgis:9.6-2.4
container_name: dockerosm_imposm hostname: db
volumes_from: environment:
- storage - POSTGRES_USER=docker
links: - POSTGRES_PASS=docker
- db:db - POSTGRES_DBNAME=gis
environment: volumes_from:
- POSTGRES_USER=docker - storage
- POSTGRES_PASS=docker # Uncomment to use the postgis database from outside the docker network
- POSTGRES_DBNAME=gis #ports:
- POSTGRES_PORT=5432 #- "35432:5432"
- POSTGRES_HOST=db healthcheck:
# seconds between 2 executions of the script test: "exit 0"
# if 0, then no update will be done, only the first initial import from the PBF
- TIME=120 imposm:
# folder for settings (with *.json and *.sql) # image: etrimaille/docker-osm:imposm-latest
- SETTINGS=settings build: docker-imposm3
# folder for caching volumes_from:
- CACHE=cache - storage
# folder for diff which has been imported depends_on:
- IMPORT_DONE=import_done db:
# folder for diff which hasn't been imported yet condition: service_healthy
- IMPORT_QUEUE=import_queue environment:
# it can be 3857 - POSTGRES_USER=docker
- SRID=4326 - POSTGRES_PASS=docker
# see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize - POSTGRES_DBNAME=gis
- OPTIMIZE=false - POSTGRES_PORT=5432
# see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - POSTGRES_HOST=db
- DBSCHEMA_PRODUCTION=public # seconds between 2 executions of the script
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables # if 0, then no update will be done, only the first initial import from the PBF
- DBSCHEMA_IMPORT=import - TIME=120
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables # folder for settings (with *.json and *.sql)
- DBSCHEMA_BACKUP=backup - SETTINGS=settings
# Install some styles if you are using the default mapping. It can be 'yes' or 'no' # folder for caching
- QGIS_STYLE=yes - CACHE=cache
# Use clip in the database # folder for diff which has been imported
- CLIP=no - IMPORT_DONE=import_done
# folder for diff which hasn't been imported yet
- IMPORT_QUEUE=import_queue
# it can be 3857
- SRID=4326
# see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
- OPTIMIZE=false
# see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_PRODUCTION=public
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_IMPORT=import
# http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
- DBSCHEMA_BACKUP=backup
# Install some styles if you are using the default mapping. It can be 'yes' or 'no'
- QGIS_STYLE=yes
# Use clip in the database
- CLIP=no
osmupdate: osmupdate:
# image: etrimaille/docker-osm:osmupdate-latest build: docker-osmupdate
build: docker-osmupdate volumes_from:
container_name: dockerosm_osmupdate - storage
volumes_from: environment:
- storage # These are all currently the defaults but listed here for your
environment: # convenience if you want to change them
# These are all currently the defaults but listed here for your # the maximum time range to assemble a cumulated changefile.
# convenience if you want to change them - MAX_DAYS=100
# the maximum time range to assemble a cumulated changefile. # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
- MAX_DAYS=100 - DIFF=sporadic
# osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. # argument to determine the maximum number of parallely processed changefiles.
- DIFF=sporadic - MAX_MERGE=7
# argument to determine the maximum number of parallely processed changefiles. # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
- MAX_MERGE=7 - COMPRESSION_LEVEL=1
# define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) # change the URL to use a custom URL to fetch regional file updates.
- COMPRESSION_LEVEL=1 - BASE_URL=http://planet.openstreetmap.org/replication/
# change the URL to use a custom URL to fetch regional file updates. # folder for diff which hasn't been imported yet
- BASE_URL=http://planet.openstreetmap.org/replication/ - IMPORT_QUEUE=import_queue
# folder for diff which hasn't been imported yet # folder for diff which has been imported
- IMPORT_QUEUE=import_queue - IMPORT_DONE=import_done
# folder for diff which has been imported # seconds between 2 executions of the script
- IMPORT_DONE=import_done # if 0, then no update will be done, only the first initial import from the PBF
# seconds between 2 executions of the script - TIME=120
# if 0, then no update will be done, only the first initial import from the PBF
- TIME=120

8
docker-imposm3/Dockerfile
Wyświetl plik

@ -1,9 +1,9 @@
FROM golang:1.9 FROM golang:1.9
MAINTAINER Etienne Trimaille <etienne@kartoza.com> MAINTAINER Etienne Trimaille <etienne@kartoza.com>
RUN apt-get update && apt-get install -y python-pip \ RUN apt-get update && apt-get install -y python3-pip \
libprotobuf-dev libleveldb-dev libgeos-dev \ libprotobuf-dev libleveldb-dev libgeos-dev \
libpq-dev python-dev postgresql-client-9.6 python-setuptools \ libpq-dev python3-dev postgresql-client-9.6 python-setuptools \
--no-install-recommends --no-install-recommends
RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
@ -14,9 +14,9 @@ RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/impo
RUN cd src/github.com/omniscale/imposm3 && make update_version && go install ./cmd/imposm/ RUN cd src/github.com/omniscale/imposm3 && make update_version && go install ./cmd/imposm/
ADD requirements.txt /home/requirements.txt ADD requirements.txt /home/requirements.txt
RUN pip install -r /home/requirements.txt RUN pip3 install -r /home/requirements.txt
ADD importer.py /home/ ADD importer.py /home/
WORKDIR /home WORKDIR /home
CMD ["python", "-u", "/home/importer.py"] CMD ["python3", "-u", "/home/importer.py"]

14
docker-imposm3/importer.py
Wyświetl plik

@ -67,13 +67,13 @@ class Importer(object):
@staticmethod @staticmethod
def error(message): def error(message):
print >> stderr, message print(stderr.write(message))
exit() exit()
def overwrite_environment(self): def overwrite_environment(self):
"""Overwrite default values from the environment.""" """Overwrite default values from the environment."""
for key in environ.keys(): for key in list(environ.keys()):
if key in self.default.keys(): if key in list(self.default.keys()):
self.default[key] = environ[key] self.default[key] = environ[key]
def check_settings(self): def check_settings(self):
@ -205,7 +205,8 @@ class Importer(object):
self.default['POSTGRES_PASS'])) self.default['POSTGRES_PASS']))
self.cursor = connection.cursor() self.cursor = connection.cursor()
except OperationalError as e: except OperationalError as e:
print >> stderr, e print(stderr.write(e))
exit() exit()
self.postgis_uri = 'postgis://%s:%s@%s/%s' % ( self.postgis_uri = 'postgis://%s:%s@%s/%s' % (
@ -285,7 +286,7 @@ class Importer(object):
def _first_pbf_import(self): def _first_pbf_import(self):
"""Run the first PBF import into the database.""" """Run the first PBF import into the database."""
command = ['imposm3', 'import', '-diff', '-deployproduction'] command = ['imposm', 'import', '-diff', '-deployproduction']
command += ['-overwritecache', '-cachedir', self.default['CACHE']] command += ['-overwritecache', '-cachedir', self.default['CACHE']]
command += ['-srid', self.default['SRID']] command += ['-srid', self.default['SRID']]
command += ['-dbschema-production', command += ['-dbschema-production',
@ -325,7 +326,7 @@ class Importer(object):
if len(import_queue) > 0: if len(import_queue) > 0:
for diff in import_queue: for diff in import_queue:
self.info('Importing diff %s' % diff) self.info('Importing diff %s' % diff)
command = ['imposm3', 'diff'] command = ['imposm', 'diff']
command += ['-cachedir', self.default['CACHE']] command += ['-cachedir', self.default['CACHE']]
command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']] command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']]
command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']] command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
@ -337,6 +338,7 @@ class Importer(object):
command += [join(self.default['IMPORT_QUEUE'], diff)] command += [join(self.default['IMPORT_QUEUE'], diff)]
self.info(' '.join(command)) self.info(' '.join(command))
if call(command) == 0: if call(command) == 0:
move( move(
join(self.default['IMPORT_QUEUE'], diff), join(self.default['IMPORT_QUEUE'], diff),

4
docker-osmupdate/Dockerfile
Wyświetl plik

@ -13,7 +13,7 @@ ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng
# RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list # RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
RUN apt-get -y update RUN apt-get -y update
RUN apt-get -y install ca-certificates rpl pwgen RUN apt-get -y install ca-certificates rpl pwgen python3
#-------------Application Specific Stuff ---------------------------------------------------- #-------------Application Specific Stuff ----------------------------------------------------
@ -28,4 +28,4 @@ RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz
# Add the python script which will call osmupdate # Add the python script which will call osmupdate
ADD download.py /home/download.py ADD download.py /home/download.py
CMD ["python", "-u", "/home/download.py"] CMD ["python3", "-u", "/home/download.py"]

14
docker-osmupdate/download.py
Wyświetl plik

@ -50,13 +50,13 @@ class Downloader(object):
@staticmethod @staticmethod
def error(message): def error(message):
print >> stderr, message print(stderr.write(message))
exit() exit()
def overwrite_environment(self): def overwrite_environment(self):
"""Overwrite default values from the environment.""" """Overwrite default values from the environment."""
for key in environ.keys(): for key in list(environ.keys()):
if key in self.default.keys(): if key in list(self.default.keys()):
self.default[key] = environ[key] self.default[key] = environ[key]
if self.default['TIME'] == '0': if self.default['TIME'] == '0':
@ -121,7 +121,13 @@ class Downloader(object):
self.info('Timestamp from the original state file : %s' % timestamp) self.info('Timestamp from the original state file : %s' % timestamp)
# Removing some \ in the timestamp. # Removing some \ in the timestamp.
timestamp = timestamp.replace('\\', '')
try:
timestamp = timestamp.decode("utf-8").replace('\\', '')
except AttributeError:
timestamp = timestamp.replace('\\', '')
pass
return timestamp return timestamp
def download(self): def download(self):

29
pbf_downloader.py
Wyświetl plik

@ -7,7 +7,7 @@ from subprocess import call
URL = 'http://download.geofabrik.de/' URL = 'http://download.geofabrik.de/'
if len(sys.argv) < 2: if len(sys.argv) < 2:
print 'Not enough argument. "list" or a name (continent or country)' print('Not enough argument. "list" or a name (continent or country)')
exit() exit()
# The JSON file comes from https://gist.github.com/Gustry/4e14bf096cdec09a3e57 # The JSON file comes from https://gist.github.com/Gustry/4e14bf096cdec09a3e57
@ -15,15 +15,15 @@ json_data = open('countries.json').read()
data = loads(json_data) data = loads(json_data)
if sys.argv[1] == 'list': if sys.argv[1] == 'list':
for continent, countries in data.items(): for continent, countries in list(data.items()):
print continent print(continent)
for country in countries: for country in countries:
print ' ' + country print(' ' + country)
exit() exit()
else: else:
area = sys.argv[1] area = sys.argv[1]
url = None url = None
for continent, countries in data.items(): for continent, countries in list(data.items()):
if area == continent: if area == continent:
url = URL + area url = URL + area
else: else:
@ -35,27 +35,26 @@ if url:
pbf_file = url + '-latest.osm.pbf' pbf_file = url + '-latest.osm.pbf'
diff = url + '-updates/' diff = url + '-updates/'
state = diff + 'state.txt' state = diff + 'state.txt'
print 'Polygon file : ' + poly_file print('Polygon file : ' + poly_file)
print 'PBF file : ' + pbf_file print('PBF file : ' + pbf_file)
print 'Diff URL : ' + diff print('Diff URL : ' + diff)
print 'state : ' + state print('state : ' + state)
print 'Downloading PBF' print('Downloading PBF')
commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file] commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file]
call(commands) call(commands)
print 'Downloading polygon' print('Downloading polygon')
commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file] commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file]
call(commands) call(commands)
print 'Downloading state' print('Downloading state')
commands = ['wget', '-c', '-O', 'settings/country.state.txt', state] commands = ['wget', '-c', '-O', 'settings/country.state.txt', state]
call(commands) call(commands)
print 'Setting custom URL diff' print('Setting custom URL diff')
with open('settings/custom_url_diff.txt', 'w') as f: with open('settings/custom_url_diff.txt', 'w') as f:
f.write(diff) f.write(diff)
else: else:
print 'This area is unkown in geofabrik or in our script. Check with the list argument.' print('This area is unkown in geofabrik or in our script. Check with the list argument.')