Fix for python path issues due to wrong version installed in docker-o… (#53)

* Fix for python path issues due to wrong version installed in docker-osm-update * Pythonize to version 3 * Fixes for PR based on comments from Gustry
2018-06-14 12:19:06 +02:00 · 2018-06-14 12:19:06 +02:00 · b8ef57c49a
commit b8ef57c49a
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,96 +1,94 @@
-storage:
-  image: ubuntu:latest
-  container_name: dockerosm_storage
-  hostname: storage
-  volumes:
-    # These are sharable to other containers
-    - ./settings:/home/settings
-    - /home/import_done
-    - /home/import_queue
-    - /home/cache
-    #- ./import_done:/home/import_done
-    #- ./import_queue:/home/import_queue
-    #- ./cache:/home/cache
+version: '2.1'

-db:
-  # About the postgresql version, it should match in the dockerfile of docker-imposm3
-  image: kartoza/postgis:9.6-2.4
-  container_name: dockerosm_db
-  hostname: db
-  environment:
-    - POSTGRES_USER=docker
-    - POSTGRES_PASS=docker
-    - POSTGRES_DBNAME=gis
-  volumes_from:
-    - storage
-# Uncomment to use the postgis database from outside the docker network
-#  ports:
-#   - "5432:5432"
+services:
+  storage:
+    image: ubuntu:bionic
+    volumes:
+      # These are sharable to other containers
+      - ./settings:/home/settings
+      - /home/import_done
+      - /home/import_queue
+      - /home/cache

-imposm:
-  # image: etrimaille/docker-osm:imposm-latest
-  build: docker-imposm3
-  container_name: dockerosm_imposm
-  volumes_from:
-    - storage
-  links:
-    - db:db
-  environment:
-    - POSTGRES_USER=docker
-    - POSTGRES_PASS=docker
-    - POSTGRES_DBNAME=gis
-    - POSTGRES_PORT=5432
-    - POSTGRES_HOST=db
-    # seconds between 2 executions of the script
-    # if 0, then no update will be done, only the first initial import from the PBF
-    - TIME=120
-    # folder for settings (with *.json and *.sql)
-    - SETTINGS=settings
-    # folder for caching
-    - CACHE=cache
-    # folder for diff which has been imported
-    - IMPORT_DONE=import_done
-    # folder for diff which hasn't been imported yet
-    - IMPORT_QUEUE=import_queue
-    # it can be 3857
-    - SRID=4326
-    # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
-    - OPTIMIZE=false
-    # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_PRODUCTION=public
-    # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_IMPORT=import
-    # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_BACKUP=backup
-    # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
-    - QGIS_STYLE=yes
-    # Use clip in the database
-    - CLIP=no
+  db:
+    # About the postgresql version, it should match in the dockerfile of docker-imposm3
+    image: kartoza/postgis:9.6-2.4
+    hostname: db
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+    volumes_from:
+      - storage
+  # Uncomment to use the postgis database from outside the docker network
+    #ports:
+      #- "35432:5432"
+    healthcheck:
+        test: "exit 0"
+
+  imposm:
+    # image: etrimaille/docker-osm:imposm-latest
+    build: docker-imposm3
+    volumes_from:
+      - storage
+    depends_on:
+        db:
+          condition: service_healthy
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+      - POSTGRES_PORT=5432
+      - POSTGRES_HOST=db
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+      # folder for settings (with *.json and *.sql)
+      - SETTINGS=settings
+      # folder for caching
+      - CACHE=cache
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # it can be 3857
+      - SRID=4326
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
+      - OPTIMIZE=false
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_PRODUCTION=public
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_IMPORT=import
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_BACKUP=backup
+      # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
+      - QGIS_STYLE=yes
+      # Use clip in the database
+      - CLIP=no


-osmupdate:
-  # image: etrimaille/docker-osm:osmupdate-latest
-  build: docker-osmupdate
-  container_name: dockerosm_osmupdate
-  volumes_from:
-    - storage
-  environment:
-    # These are all currently the defaults but listed here for your
-    # convenience if you want to change them
-    # the maximum time range to assemble a cumulated changefile.
-    - MAX_DAYS=100
-    # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
-    - DIFF=sporadic
-    # argument to determine the maximum number of parallely processed changefiles.
-    - MAX_MERGE=7
-    # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
-    - COMPRESSION_LEVEL=1
-    # change the URL to use a custom URL to fetch regional file updates.
-    - BASE_URL=http://planet.openstreetmap.org/replication/
-    # folder for diff which hasn't been imported yet
-    - IMPORT_QUEUE=import_queue
-    # folder for diff which has been imported
-    - IMPORT_DONE=import_done
-    # seconds between 2 executions of the script
-    # if 0, then no update will be done, only the first initial import from the PBF
-    - TIME=120
+  osmupdate:
+    build: docker-osmupdate
+    volumes_from:
+      - storage
+    environment:
+      # These are all currently the defaults but listed here for your
+      # convenience if you want to change them
+      # the maximum time range to assemble a cumulated changefile.
+      - MAX_DAYS=100
+      # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+      - DIFF=sporadic
+      # argument to determine the maximum number of parallely processed changefiles.
+      - MAX_MERGE=7
+      # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
+      - COMPRESSION_LEVEL=1
+      # change the URL to use a custom URL to fetch regional file updates.
+      - BASE_URL=http://planet.openstreetmap.org/replication/
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+
--- a/docker-imposm3/Dockerfile
+++ b/docker-imposm3/Dockerfile
@ -1,9 +1,9 @@
 FROM golang:1.9
 MAINTAINER Etienne Trimaille <etienne@kartoza.com>

-RUN apt-get update && apt-get install -y python-pip \
+RUN apt-get update && apt-get install -y python3-pip \
      libprotobuf-dev libleveldb-dev libgeos-dev \
-      libpq-dev python-dev postgresql-client-9.6 python-setuptools \
+      libpq-dev python3-dev postgresql-client-9.6 python-setuptools \
      --no-install-recommends

 RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
@ -14,9 +14,9 @@ RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/impo
 RUN cd src/github.com/omniscale/imposm3 && make update_version && go install ./cmd/imposm/

 ADD requirements.txt /home/requirements.txt
-RUN pip install -r /home/requirements.txt
+RUN pip3 install -r /home/requirements.txt

 ADD importer.py /home/

 WORKDIR /home
-CMD ["python", "-u", "/home/importer.py"]
+CMD ["python3", "-u", "/home/importer.py"]
--- a/docker-imposm3/importer.py
+++ b/docker-imposm3/importer.py
@ -67,13 +67,13 @@ class Importer(object):

    @staticmethod
    def error(message):
-        print >> stderr, message
+        print(stderr.write(message))
        exit()

    def overwrite_environment(self):
        """Overwrite default values from the environment."""
-        for key in environ.keys():
-            if key in self.default.keys():
+        for key in list(environ.keys()):
+            if key in list(self.default.keys()):
                self.default[key] = environ[key]

    def check_settings(self):
@ -205,7 +205,8 @@ class Importer(object):
                    self.default['POSTGRES_PASS']))
            self.cursor = connection.cursor()
        except OperationalError as e:
-            print >> stderr, e
+            print(stderr.write(e))
+
            exit()

        self.postgis_uri = 'postgis://%s:%s@%s/%s' % (
@ -285,7 +286,7 @@ class Importer(object):

    def _first_pbf_import(self):
        """Run the first PBF import into the database."""
-        command = ['imposm3', 'import', '-diff', '-deployproduction']
+        command = ['imposm', 'import', '-diff', '-deployproduction']
        command += ['-overwritecache', '-cachedir', self.default['CACHE']]
        command += ['-srid', self.default['SRID']]
        command += ['-dbschema-production',
@ -325,7 +326,7 @@ class Importer(object):
            if len(import_queue) > 0:
                for diff in import_queue:
                    self.info('Importing diff %s' % diff)
-                    command = ['imposm3', 'diff']
+                    command = ['imposm', 'diff']
                    command += ['-cachedir', self.default['CACHE']]
                    command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']]
                    command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
@ -337,6 +338,7 @@ class Importer(object):
                    command += [join(self.default['IMPORT_QUEUE'], diff)]

                    self.info(' '.join(command))
+
                    if call(command) == 0:
                        move(
                            join(self.default['IMPORT_QUEUE'], diff),
--- a/docker-osmupdate/Dockerfile
+++ b/docker-osmupdate/Dockerfile
@ -13,7 +13,7 @@ ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng

 # RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
 RUN apt-get -y update
-RUN apt-get -y install ca-certificates rpl pwgen
+RUN apt-get -y install ca-certificates rpl pwgen python3

 #-------------Application Specific Stuff ----------------------------------------------------

@ -28,4 +28,4 @@ RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz
 # Add the python script which will call osmupdate
 ADD download.py /home/download.py

-CMD ["python", "-u", "/home/download.py"]
+CMD ["python3", "-u", "/home/download.py"]
--- a/docker-osmupdate/download.py
+++ b/docker-osmupdate/download.py
@ -50,13 +50,13 @@ class Downloader(object):

    @staticmethod
    def error(message):
-        print >> stderr, message
+        print(stderr.write(message))
        exit()

    def overwrite_environment(self):
        """Overwrite default values from the environment."""
-        for key in environ.keys():
-            if key in self.default.keys():
+        for key in list(environ.keys()):
+            if key in list(self.default.keys()):
                self.default[key] = environ[key]

        if self.default['TIME'] == '0':
@ -121,7 +121,13 @@ class Downloader(object):
                self.info('Timestamp from the original state file : %s' % timestamp)

        # Removing some \ in the timestamp.
-        timestamp = timestamp.replace('\\', '')
+
+        try:
+            timestamp = timestamp.decode("utf-8").replace('\\', '')
+        except AttributeError:
+            timestamp = timestamp.replace('\\', '')
+            pass
+
        return timestamp

    def download(self):
--- a/pbf_downloader.py
+++ b/pbf_downloader.py
@ -7,7 +7,7 @@ from subprocess import call
 URL = 'http://download.geofabrik.de/'

 if len(sys.argv) < 2:
-    print 'Not enough argument. "list" or a name (continent or country)'
+    print('Not enough argument. "list" or a name (continent or country)')
    exit()

 # The JSON file comes from https://gist.github.com/Gustry/4e14bf096cdec09a3e57
@ -15,15 +15,15 @@ json_data = open('countries.json').read()
 data = loads(json_data)

 if sys.argv[1] == 'list':
-    for continent, countries in data.items():
-        print continent
+    for continent, countries in list(data.items()):
+        print(continent)
        for country in countries:
-            print '     ' + country
+            print('     ' + country)
    exit()
 else:
    area = sys.argv[1]
    url = None
-    for continent, countries in data.items():
+    for continent, countries in list(data.items()):
        if area == continent:
            url = URL + area
        else:
@ -35,27 +35,26 @@ if url:
    pbf_file = url + '-latest.osm.pbf'
    diff = url + '-updates/'
    state = diff + 'state.txt'
-    print 'Polygon file : ' + poly_file
-    print 'PBF file : ' + pbf_file
-    print 'Diff URL : ' + diff
-    print 'state : ' + state
+    print('Polygon file : ' + poly_file)
+    print('PBF file : ' + pbf_file)
+    print('Diff URL : ' + diff)
+    print('state : ' + state)

-    print 'Downloading PBF'
+    print('Downloading PBF')
    commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file]
    call(commands)

-    print 'Downloading polygon'
+    print('Downloading polygon')
    commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file]
    call(commands)

-    print 'Downloading state'
+    print('Downloading state')
    commands = ['wget', '-c', '-O', 'settings/country.state.txt', state]
    call(commands)

-    print 'Setting custom URL diff'
+    print('Setting custom URL diff')
    with open('settings/custom_url_diff.txt', 'w') as f:
        f.write(diff)

 else:
-    print 'This area is unkown in geofabrik or in our script. Check with the list argument.'
-
+    print('This area is unkown in geofabrik or in our script. Check with the list argument.')