Fix for python path issues due to wrong version installed in docker-o… (#53)

* Fix for python path issues due to wrong version installed in docker-osm-update

* Pythonize to version 3

* Fixes for PR based on comments from Gustry

docker-compose.yml

@@ -1,21 +1,18 @@
+version: '2.1'
+
+services:
   storage:
-  image: ubuntu:latest
-  container_name: dockerosm_storage
-  hostname: storage
+    image: ubuntu:bionic
     volumes:
       # These are sharable to other containers
       - ./settings:/home/settings
       - /home/import_done
       - /home/import_queue
       - /home/cache
-    #- ./import_done:/home/import_done
-    #- ./import_queue:/home/import_queue
-    #- ./cache:/home/cache
 
   db:
     # About the postgresql version, it should match in the dockerfile of docker-imposm3
     image: kartoza/postgis:9.6-2.4
-  container_name: dockerosm_db
     hostname: db
     environment:
       - POSTGRES_USER=docker
@@ -25,16 +22,18 @@ db:
       - storage
   # Uncomment to use the postgis database from outside the docker network
     #ports:
-#   - "5432:5432"
+      #- "35432:5432"
+    healthcheck:
+        test: "exit 0"
 
   imposm:
     # image: etrimaille/docker-osm:imposm-latest
     build: docker-imposm3
-  container_name: dockerosm_imposm
     volumes_from:
       - storage
-  links:
-    - db:db
+    depends_on:
+        db:
+          condition: service_healthy
     environment:
       - POSTGRES_USER=docker
       - POSTGRES_PASS=docker
@@ -69,9 +68,7 @@ imposm:
 
 
   osmupdate:
-  # image: etrimaille/docker-osm:osmupdate-latest
     build: docker-osmupdate
-  container_name: dockerosm_osmupdate
     volumes_from:
       - storage
     environment:
@@ -94,3 +91,4 @@ osmupdate:
       # seconds between 2 executions of the script
       # if 0, then no update will be done, only the first initial import from the PBF
       - TIME=120
+

docker-imposm3/Dockerfile

@@ -1,9 +1,9 @@
 FROM golang:1.9
 MAINTAINER Etienne Trimaille <etienne@kartoza.com>
 
-RUN apt-get update && apt-get install -y python-pip \
+RUN apt-get update && apt-get install -y python3-pip \
       libprotobuf-dev libleveldb-dev libgeos-dev \
-      libpq-dev python-dev postgresql-client-9.6 python-setuptools \
+      libpq-dev python3-dev postgresql-client-9.6 python-setuptools \
       --no-install-recommends
 
 RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
@@ -14,9 +14,9 @@ RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/impo
 RUN cd src/github.com/omniscale/imposm3 && make update_version && go install ./cmd/imposm/
 
 ADD requirements.txt /home/requirements.txt
-RUN pip install -r /home/requirements.txt
+RUN pip3 install -r /home/requirements.txt
 
 ADD importer.py /home/
 
 WORKDIR /home
-CMD ["python", "-u", "/home/importer.py"]
+CMD ["python3", "-u", "/home/importer.py"]

docker-imposm3/importer.py

@@ -67,13 +67,13 @@ class Importer(object):
 
     @staticmethod
     def error(message):
-        print >> stderr, message
+        print(stderr.write(message))
         exit()
 
     def overwrite_environment(self):
         """Overwrite default values from the environment."""
-        for key in environ.keys():
-            if key in self.default.keys():
+        for key in list(environ.keys()):
+            if key in list(self.default.keys()):
                 self.default[key] = environ[key]
 
     def check_settings(self):
@@ -205,7 +205,8 @@ class Importer(object):
                     self.default['POSTGRES_PASS']))
             self.cursor = connection.cursor()
         except OperationalError as e:
-            print >> stderr, e
+            print(stderr.write(e))
+
             exit()
 
         self.postgis_uri = 'postgis://%s:%s@%s/%s' % (
@@ -285,7 +286,7 @@ class Importer(object):
 
     def _first_pbf_import(self):
         """Run the first PBF import into the database."""
-        command = ['imposm3', 'import', '-diff', '-deployproduction']
+        command = ['imposm', 'import', '-diff', '-deployproduction']
         command += ['-overwritecache', '-cachedir', self.default['CACHE']]
         command += ['-srid', self.default['SRID']]
         command += ['-dbschema-production',
@@ -325,7 +326,7 @@ class Importer(object):
             if len(import_queue) > 0:
                 for diff in import_queue:
                     self.info('Importing diff %s' % diff)
-                    command = ['imposm3', 'diff']
+                    command = ['imposm', 'diff']
                     command += ['-cachedir', self.default['CACHE']]
                     command += ['-dbschema-production', self.default['DBSCHEMA_PRODUCTION']]
                     command += ['-dbschema-import', self.default['DBSCHEMA_IMPORT']]
@@ -337,6 +338,7 @@ class Importer(object):
                     command += [join(self.default['IMPORT_QUEUE'], diff)]
 
                     self.info(' '.join(command))
+
                     if call(command) == 0:
                         move(
                             join(self.default['IMPORT_QUEUE'], diff),

docker-osmupdate/Dockerfile

@@ -13,7 +13,7 @@ ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng
 
 # RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
 RUN apt-get -y update
-RUN apt-get -y install ca-certificates rpl pwgen
+RUN apt-get -y install ca-certificates rpl pwgen python3
 
 #-------------Application Specific Stuff ----------------------------------------------------
 
@@ -28,4 +28,4 @@ RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz
 # Add the python script which will call osmupdate
 ADD download.py /home/download.py
 
-CMD ["python", "-u", "/home/download.py"]
+CMD ["python3", "-u", "/home/download.py"]

docker-osmupdate/download.py

@@ -50,13 +50,13 @@ class Downloader(object):
 
     @staticmethod
     def error(message):
-        print >> stderr, message
+        print(stderr.write(message))
         exit()
 
     def overwrite_environment(self):
         """Overwrite default values from the environment."""
-        for key in environ.keys():
-            if key in self.default.keys():
+        for key in list(environ.keys()):
+            if key in list(self.default.keys()):
                 self.default[key] = environ[key]
 
         if self.default['TIME'] == '0':
@@ -121,7 +121,13 @@ class Downloader(object):
                 self.info('Timestamp from the original state file : %s' % timestamp)
 
         # Removing some \ in the timestamp.
+
+        try:
+            timestamp = timestamp.decode("utf-8").replace('\\', '')
+        except AttributeError:
             timestamp = timestamp.replace('\\', '')
+            pass
+
         return timestamp
 
     def download(self):

pbf_downloader.py

@@ -7,7 +7,7 @@ from subprocess import call
 URL = 'http://download.geofabrik.de/'
 
 if len(sys.argv) < 2:
-    print 'Not enough argument. "list" or a name (continent or country)'
+    print('Not enough argument. "list" or a name (continent or country)')
     exit()
 
 # The JSON file comes from https://gist.github.com/Gustry/4e14bf096cdec09a3e57
@@ -15,15 +15,15 @@ json_data = open('countries.json').read()
 data = loads(json_data)
 
 if sys.argv[1] == 'list':
-    for continent, countries in data.items():
-        print continent
+    for continent, countries in list(data.items()):
+        print(continent)
         for country in countries:
-            print '     ' + country
+            print('     ' + country)
     exit()
 else:
     area = sys.argv[1]
     url = None
-    for continent, countries in data.items():
+    for continent, countries in list(data.items()):
         if area == continent:
             url = URL + area
         else:
@@ -35,27 +35,26 @@ if url:
     pbf_file = url + '-latest.osm.pbf'
     diff = url + '-updates/'
     state = diff + 'state.txt'
-    print 'Polygon file : ' + poly_file
-    print 'PBF file : ' + pbf_file
-    print 'Diff URL : ' + diff
-    print 'state : ' + state
+    print('Polygon file : ' + poly_file)
+    print('PBF file : ' + pbf_file)
+    print('Diff URL : ' + diff)
+    print('state : ' + state)
 
-    print 'Downloading PBF'
+    print('Downloading PBF')
     commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file]
     call(commands)
 
-    print 'Downloading polygon'
+    print('Downloading polygon')
     commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file]
     call(commands)
 
-    print 'Downloading state'
+    print('Downloading state')
     commands = ['wget', '-c', '-O', 'settings/country.state.txt', state]
     call(commands)
 
-    print 'Setting custom URL diff'
+    print('Setting custom URL diff')
     with open('settings/custom_url_diff.txt', 'w') as f:
         f.write(diff)
 
 else:
-    print 'This area is unkown in geofabrik or in our script. Check with the list argument.'
-
+    print('This area is unkown in geofabrik or in our script. Check with the list argument.')