diff --git a/docker-compose.yml b/docker-compose.yml index 7306486..4de3529 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,96 +1,94 @@ -storage: - image: ubuntu:latest - container_name: dockerosm_storage - hostname: storage - volumes: - # These are sharable to other containers - - ./settings:/home/settings - - /home/import_done - - /home/import_queue - - /home/cache - #- ./import_done:/home/import_done - #- ./import_queue:/home/import_queue - #- ./cache:/home/cache +version: '2.1' -db: - # About the postgresql version, it should match in the dockerfile of docker-imposm3 - image: kartoza/postgis:9.6-2.4 - container_name: dockerosm_db - hostname: db - environment: - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - - POSTGRES_DBNAME=gis - volumes_from: - - storage -# Uncomment to use the postgis database from outside the docker network -# ports: -# - "5432:5432" +services: + storage: + image: ubuntu:bionic + volumes: + # These are sharable to other containers + - ./settings:/home/settings + - /home/import_done + - /home/import_queue + - /home/cache -imposm: - # image: etrimaille/docker-osm:imposm-latest - build: docker-imposm3 - container_name: dockerosm_imposm - volumes_from: - - storage - links: - - db:db - environment: - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - - POSTGRES_DBNAME=gis - - POSTGRES_PORT=5432 - - POSTGRES_HOST=db - # seconds between 2 executions of the script - # if 0, then no update will be done, only the first initial import from the PBF - - TIME=120 - # folder for settings (with *.json and *.sql) - - SETTINGS=settings - # folder for caching - - CACHE=cache - # folder for diff which has been imported - - IMPORT_DONE=import_done - # folder for diff which hasn't been imported yet - - IMPORT_QUEUE=import_queue - # it can be 3857 - - SRID=4326 - # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize - - OPTIMIZE=false - # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_PRODUCTION=public - # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_IMPORT=import - # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_BACKUP=backup - # Install some styles if you are using the default mapping. It can be 'yes' or 'no' - - QGIS_STYLE=yes - # Use clip in the database - - CLIP=no + db: + # About the postgresql version, it should match in the dockerfile of docker-imposm3 + image: kartoza/postgis:9.6-2.4 + hostname: db + environment: + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - POSTGRES_DBNAME=gis + volumes_from: + - storage + # Uncomment to use the postgis database from outside the docker network + #ports: + #- "35432:5432" + healthcheck: + test: "exit 0" + + imposm: + # image: etrimaille/docker-osm:imposm-latest + build: docker-imposm3 + volumes_from: + - storage + depends_on: + db: + condition: service_healthy + environment: + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - POSTGRES_DBNAME=gis + - POSTGRES_PORT=5432 + - POSTGRES_HOST=db + # seconds between 2 executions of the script + # if 0, then no update will be done, only the first initial import from the PBF + - TIME=120 + # folder for settings (with *.json and *.sql) + - SETTINGS=settings + # folder for caching + - CACHE=cache + # folder for diff which has been imported + - IMPORT_DONE=import_done + # folder for diff which hasn't been imported yet + - IMPORT_QUEUE=import_queue + # it can be 3857 + - SRID=4326 + # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize + - OPTIMIZE=false + # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_PRODUCTION=public + # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_IMPORT=import + # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_BACKUP=backup + # Install some styles if you are using the default mapping. It can be 'yes' or 'no' + - QGIS_STYLE=yes + # Use clip in the database + - CLIP=no -osmupdate: - # image: etrimaille/docker-osm:osmupdate-latest - build: docker-osmupdate - container_name: dockerosm_osmupdate - volumes_from: - - storage - environment: - # These are all currently the defaults but listed here for your - # convenience if you want to change them - # the maximum time range to assemble a cumulated changefile. - - MAX_DAYS=100 - # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. - - DIFF=sporadic - # argument to determine the maximum number of parallely processed changefiles. - - MAX_MERGE=7 - # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) - - COMPRESSION_LEVEL=1 - # change the URL to use a custom URL to fetch regional file updates. - - BASE_URL=http://planet.openstreetmap.org/replication/ - # folder for diff which hasn't been imported yet - - IMPORT_QUEUE=import_queue - # folder for diff which has been imported - - IMPORT_DONE=import_done - # seconds between 2 executions of the script - # if 0, then no update will be done, only the first initial import from the PBF - - TIME=120 + osmupdate: + build: docker-osmupdate + volumes_from: + - storage + environment: + # These are all currently the defaults but listed here for your + # convenience if you want to change them + # the maximum time range to assemble a cumulated changefile. + - MAX_DAYS=100 + # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. + - DIFF=sporadic + # argument to determine the maximum number of parallely processed changefiles. + - MAX_MERGE=7 + # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) + - COMPRESSION_LEVEL=1 + # change the URL to use a custom URL to fetch regional file updates. + - BASE_URL=http://planet.openstreetmap.org/replication/ + # folder for diff which hasn't been imported yet + - IMPORT_QUEUE=import_queue + # folder for diff which has been imported + - IMPORT_DONE=import_done + # seconds between 2 executions of the script + # if 0, then no update will be done, only the first initial import from the PBF + - TIME=120 + diff --git a/docker-imposm3/Dockerfile b/docker-imposm3/Dockerfile index 7a443a7..11e81dd 100644 --- a/docker-imposm3/Dockerfile +++ b/docker-imposm3/Dockerfile @@ -1,9 +1,9 @@ FROM golang:1.9 MAINTAINER Etienne Trimaille -RUN apt-get update && apt-get install -y python-pip \ +RUN apt-get update && apt-get install -y python3-pip \ libprotobuf-dev libleveldb-dev libgeos-dev \ - libpq-dev python-dev postgresql-client-9.6 python-setuptools \ + libpq-dev python3-dev postgresql-client-9.6 python-setuptools \ --no-install-recommends RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so @@ -14,9 +14,9 @@ RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/impo RUN cd src/github.com/omniscale/imposm3 && make update_version && go install ./cmd/imposm/ ADD requirements.txt /home/requirements.txt -RUN pip install -r /home/requirements.txt +RUN pip3 install -r /home/requirements.txt ADD importer.py /home/ WORKDIR /home -CMD ["python", "-u", "/home/importer.py"] +CMD ["python3", "-u", "/home/importer.py"] diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index c6527bf..4138ec8 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -67,13 +67,13 @@ class Importer(object): @staticmethod def error(message): - print >> stderr, message + print(stderr.write(message)) exit() def overwrite_environment(self): """Overwrite default values from the environment.""" - for key in environ.keys(): - if key in self.default.keys(): + for key in list(environ.keys()): + if key in list(self.default.keys()): self.default[key] = environ[key] def check_settings(self): @@ -205,7 +205,8 @@ class Importer(object): self.default['POSTGRES_PASS'])) self.cursor = connection.cursor() except OperationalError as e: - print >> stderr, e + print(stderr.write(e)) + exit() self.postgis_uri = 'postgis://%s:%s@%s/%s' % ( @@ -285,7 +286,7 @@ class Importer(object): def _first_pbf_import(self): """Run the first PBF import into the database.""" - command = ['imposm3', 'import', '-diff', '-deployproduction'] + command = ['imposm', 'import', '-diff', '-deployproduction'] command += ['-overwritecache', '-cachedir', self.default['CACHE']] command += ['-srid', self.default['SRID']] command += ['-dbschema-production', @@ -325,7 +326,7 @@ class Importer(object): if len(import_queue) > 0: for diff in import_queue: self.info('Importing diff %s' % diff) - command = ['imposm3', 'diff'] + command = ['imposm', 'diff'] command += ['-cachedir', self.default['CACHE']] command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']] command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']] @@ -337,6 +338,7 @@ class Importer(object): command += [join(self.default['IMPORT_QUEUE'], diff)] self.info(' '.join(command)) + if call(command) == 0: move( join(self.default['IMPORT_QUEUE'], diff), diff --git a/docker-osmupdate/Dockerfile b/docker-osmupdate/Dockerfile index f83ca3f..3ea45aa 100644 --- a/docker-osmupdate/Dockerfile +++ b/docker-osmupdate/Dockerfile @@ -13,7 +13,7 @@ ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng # RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list RUN apt-get -y update -RUN apt-get -y install ca-certificates rpl pwgen +RUN apt-get -y install ca-certificates rpl pwgen python3 #-------------Application Specific Stuff ---------------------------------------------------- @@ -28,4 +28,4 @@ RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz # Add the python script which will call osmupdate ADD download.py /home/download.py -CMD ["python", "-u", "/home/download.py"] \ No newline at end of file +CMD ["python3", "-u", "/home/download.py"] \ No newline at end of file diff --git a/docker-osmupdate/download.py b/docker-osmupdate/download.py index 121734e..047a86d 100644 --- a/docker-osmupdate/download.py +++ b/docker-osmupdate/download.py @@ -50,13 +50,13 @@ class Downloader(object): @staticmethod def error(message): - print >> stderr, message + print(stderr.write(message)) exit() def overwrite_environment(self): """Overwrite default values from the environment.""" - for key in environ.keys(): - if key in self.default.keys(): + for key in list(environ.keys()): + if key in list(self.default.keys()): self.default[key] = environ[key] if self.default['TIME'] == '0': @@ -121,7 +121,13 @@ class Downloader(object): self.info('Timestamp from the original state file : %s' % timestamp) # Removing some \ in the timestamp. - timestamp = timestamp.replace('\\', '') + + try: + timestamp = timestamp.decode("utf-8").replace('\\', '') + except AttributeError: + timestamp = timestamp.replace('\\', '') + pass + return timestamp def download(self): diff --git a/pbf_downloader.py b/pbf_downloader.py index 5169095..3e36922 100644 --- a/pbf_downloader.py +++ b/pbf_downloader.py @@ -7,7 +7,7 @@ from subprocess import call URL = 'http://download.geofabrik.de/' if len(sys.argv) < 2: - print 'Not enough argument. "list" or a name (continent or country)' + print('Not enough argument. "list" or a name (continent or country)') exit() # The JSON file comes from https://gist.github.com/Gustry/4e14bf096cdec09a3e57 @@ -15,15 +15,15 @@ json_data = open('countries.json').read() data = loads(json_data) if sys.argv[1] == 'list': - for continent, countries in data.items(): - print continent + for continent, countries in list(data.items()): + print(continent) for country in countries: - print ' ' + country + print(' ' + country) exit() else: area = sys.argv[1] url = None - for continent, countries in data.items(): + for continent, countries in list(data.items()): if area == continent: url = URL + area else: @@ -35,27 +35,26 @@ if url: pbf_file = url + '-latest.osm.pbf' diff = url + '-updates/' state = diff + 'state.txt' - print 'Polygon file : ' + poly_file - print 'PBF file : ' + pbf_file - print 'Diff URL : ' + diff - print 'state : ' + state + print('Polygon file : ' + poly_file) + print('PBF file : ' + pbf_file) + print('Diff URL : ' + diff) + print('state : ' + state) - print 'Downloading PBF' + print('Downloading PBF') commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file] call(commands) - print 'Downloading polygon' + print('Downloading polygon') commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file] call(commands) - print 'Downloading state' + print('Downloading state') commands = ['wget', '-c', '-O', 'settings/country.state.txt', state] call(commands) - print 'Setting custom URL diff' + print('Setting custom URL diff') with open('settings/custom_url_diff.txt', 'w') as f: f.write(diff) else: - print 'This area is unkown in geofabrik or in our script. Check with the list argument.' - + print('This area is unkown in geofabrik or in our script. Check with the list argument.')