Merge pull request #24 from Gustry/improvments

Improvments

.gitignore

@@ -1,7 +1,11 @@
 .idea
 *.*~
+cache
+import_done
+import_queue
 settings/*.pbf
 settings/*.state.txt
 settings/*.poly
 settings/last.state.txt
 settings/custom_url_diff.txt
+settings/timestamp.txt

Makefile

@@ -0,0 +1,51 @@
+PROJECT_ID := dockerosm
+
+
+build:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Building in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) build
+
+run:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d --no-recreate
+
+rundev:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in DEVELOPMENT mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up
+
+kill:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Killing in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) kill
+
+rm: kill
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing production instance!!! "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) rm
+
+ipdb:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Database's IP"
+	@echo "------------------------------------------------------------------"
+	@docker inspect $(PROJECT_ID)_db_1 | grep '"IPAddress"' | head -1 | cut -d '"' -f 4
+
+timestamp:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Timestamp"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_imposm_1 cat /home/settings/timestamp.txt

docker-compose.yml

@@ -12,6 +12,7 @@ storage:
     #- ./cache:/home/cache
 
 db:
+  # About the postgresql version, it should match in the dockerfile of docker-imposm3
   image: kartoza/postgis:9.4-2.1
   hostname: db
   environment:
@@ -30,27 +31,29 @@ imposm:
     - PASSWORD=docker
     - PORT=5432
     - HOST=db
-    - DATEBASE=gis
+    - DATABASE=gis
     # seconds between 2 executions of the script
     - TIME=120
     # folder for settings (with *.json and *.sql)
-    - SETTINGS = settings
+    - SETTINGS=settings
     # folder for caching
-    - CACHE = cache
+    - CACHE=cache
     # folder for diff which has been imported
-    - IMPORT_DONE = import_done
+    - IMPORT_DONE=import_done
     # folder for diff which hasn't been imported yet
-    - IMPORT_QUEUE = import_queue
+    - IMPORT_QUEUE=import_queue
     # it can be 3857
-    - SRID = 4326
+    - SRID=4326
     # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
-    - OPTIMIZE = false
+    - OPTIMIZE=false
     # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_PRODUCTION = public
+    - DBSCHEMA_PRODUCTION=public
     # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_IMPORT = import
+    - DBSCHEMA_IMPORT=import
     # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-    - DBSCHEMA_BACKUP = backup
+    - DBSCHEMA_BACKUP=backup
+    # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
+    - QGIS_STYLE=yes
 
 
 osmupdate:
@@ -61,18 +64,18 @@ osmupdate:
     # These are all currently the defaults but listed here for your
     # convenience if you want to change them
     # the maximum time range to assamble a cumulated changefile.
-    - MAX_DAYS = 100
+    - MAX_DAYS=100
     # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
-    - DIFF = sporadic
+    - DIFF=sporadic
     # argument to determine the maximum number of parallely processed changefiles.
-    - MAX_MERGE = 7
+    - MAX_MERGE=7
     # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
-    - COMPRESSION_LEVEL = 1
+    - COMPRESSION_LEVEL=1
     # change the URL to use a custom URL to fetch regional file updates.
-    - BASE_URL = http://planet.openstreetmap.org/replication/
+    - BASE_URL=http://planet.openstreetmap.org/replication/
     # folder for diff which hasn't been imported yet
-    - IMPORT_QUEUE = import_queue
+    - IMPORT_QUEUE=import_queue
     # folder for diff which has been imported
-    - IMPORT_DONE = import_done
+    - IMPORT_DONE=import_done
     # seconds between 2 executions of the script
-    - TIME = 120
+    - TIME=120

docker-imposm3/Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.4
+FROM golang:1.5
 MAINTAINER Etienne Trimaille <etienne@kartoza.com>
 
 RUN apt-get update && apt-get install -y python-pip \
       libprotobuf-dev libleveldb-dev libgeos-dev \
-      libpq-dev python-dev \
+      libpq-dev python-dev postgresql-client-9.4 \
       --no-install-recommends
 
 RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
@@ -11,7 +11,7 @@ RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so
 WORKDIR $GOPATH
 RUN go get github.com/tools/godep
 RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/imposm3
-RUN cd src/github.com/omniscale/imposm3 && godep go install ./...
+RUN cd src/github.com/omniscale/imposm3 && godep go install ./
 
 ADD requirements.txt /home/requirements.txt
 RUN pip install -r /home/requirements.txt

docker-imposm3/importer.py

@@ -44,17 +44,23 @@ default = {
     'OPTIMIZE': 'false',
     'DBSCHEMA_PRODUCTION': 'public',
     'DBSCHEMA_IMPORT': 'import',
-    'DBSCHEMA_BACKUP': 'backup'
+    'DBSCHEMA_BACKUP': 'backup',
+    'QGIS_STYLE': 'yes'
 }
 
 # Check if we overwrite default values.
-for key in default.keys():
+for key in environ.keys():
-    if key in environ:
+    if key in default.keys():
         default[key] = environ[key]
 
 # Check valid SRID.
 if default['SRID'] not in ['4326', '3857']:
-    print >> stderr, 'SRID not supported : %s' % default['srid']
+    print >> stderr, 'SRID not supported : %s' % default['SRID']
+    exit()
+
+# Check valid QGIS_STYLE.
+if default['QGIS_STYLE'] not in ['yes', 'no']:
+    print >> stderr, 'QGIS_STYLE not supported : %s' % default['QGIS_STYLE']
     exit()
 
 # Check folders.
@@ -70,13 +76,11 @@ for folder in folders:
         exit()
 
 # Test files
-state_file = None
 osm_file = None
 mapping_file = None
 post_import_file = None
+qgis_style = None
 for f in listdir(default['SETTINGS']):
-    if f == 'last.state.txt':
-        state_file = join(default['SETTINGS'], f)
 
     if f.endswith('.pbf'):
         osm_file = join(default['SETTINGS'], f)
@@ -84,17 +88,16 @@ for f in listdir(default['SETTINGS']):
     if f.endswith('.json'):
         mapping_file = join(default['SETTINGS'], f)
 
-    if f.endswith('.sql'):
+    if f == 'post-pbf-import.sql':
         post_import_file = join(default['SETTINGS'], f)
 
+    if f == 'qgis_style.sql':
+        qgis_style = join(default['SETTINGS'], f)
+
 if not osm_file:
     print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
     exit()
 
-if not state_file:
-    print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
-    exit()
-
 if not mapping_file:
     print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
     exit()
@@ -104,6 +107,14 @@ if not post_import_file:
 else:
     print '%s detected for post import.' % post_import_file
 
+if not qgis_style and default['QGIS_STYLE'] == 'yes':
+    print >> stderr, 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % default['SETTINGS']
+    exit()
+elif qgis_style and default['QGIS_STYLE']:
+    print '%s detected for QGIS styling.' % qgis_style
+else:
+    print 'Not using QGIS default styles.'
+
 # In docker-compose, we should wait for the DB is ready.
 print 'The checkup is OK. The container will continue soon, after the database.'
 sleep(45)
@@ -157,9 +168,27 @@ if osm_tables < 1:
     else:
         print 'Import PBF successful : %s' % osm_file
 
+    if post_import_file or qgis_style:
+        # Set the password for psql
+        environ['PGPASSWORD'] = default['PASSWORD']
+
     if post_import_file:
-        for sql in open(post_import_file):
+        print 'Running the post import SQL file.'
-            cursor.execute(sql)
+        command = ['psql']
+        command += ['-h', default['HOST']]
+        command += ['-U', default['USER']]
+        command += ['-d', default['DATABASE']]
+        command += ['-f', post_import_file]
+        call(command)
+
+    if qgis_style:
+        'Installing QGIS styles.'
+        command = ['psql']
+        command += ['-h', default['HOST']]
+        command += ['-U', default['USER']]
+        command += ['-d', default['DATABASE']]
+        command += ['-f', qgis_style]
+        call(command)
 else:
     print 'The database is not empty. Let\'s import only diff files.'
 
@@ -185,6 +214,14 @@ while True:
                 move(
                     join(default['IMPORT_QUEUE'], diff),
                     join(default['IMPORT_DONE'], diff))
+
+                # Update the timestamp in the file.
+                database_timestamp = diff.split('.')[0].split('->-')[1]
+                file_path = join(default['SETTINGS'], 'timestamp.txt')
+                timestamp_file = open(file_path, 'w')
+                timestamp_file.write('%s\n' % database_timestamp)
+                timestamp_file.close()
+
                 print 'Import diff successful : %s' % diff
             else:
                 print >> stderr, 'An error occured in imposm with a diff.'

docker-osmupdate/Dockerfile

@@ -21,7 +21,9 @@ RUN apt-get -y install ca-certificates rpl pwgen
 RUN apt-get -y install osmctools wget gzip gcc libc-dev zlib1g-dev
 WORKDIR /home
 ADD osmupdate.c /home/osmupdate.c
+ADD osmconvert.c /home/osmconvert.c
 RUN gcc -x c - -o osmupdate osmupdate.c
+RUN gcc -x c - -O3 -o osmconvert osmconvert.c -lz
 
 # Add the python script which will call osmupdate
 ADD download.py /home/download.py

docker-osmupdate/download.py

@@ -22,15 +22,11 @@
 from os.path import exists, join, isabs, abspath
 from os import listdir, environ
 from sys import exit
-from subprocess import call
+from subprocess import call, Popen, PIPE
 from datetime import datetime
 from time import sleep
 from sys import stderr
 
-# In docker-compose, we should wait for the DB is ready.
-print 'The container will start soon, after the database.'
-sleep(45)
-
 # Default values which can be overwritten.
 default = {
     'MAX_DAYS': '100',
@@ -44,8 +40,8 @@ default = {
     'TIME': 120,
 }
 
-for key in default.keys():
+for key in environ.keys():
-    if key in environ:
+    if key in default.keys():
         default[key] = environ[key]
 
 # Folders
@@ -61,19 +57,12 @@ for folder in folders:
         exit()
 
 # Test files
-state_file = None
 osm_file = None
-poly_file = None
 for f in listdir(default['SETTINGS']):
-    if f == 'last.state.txt':
-        state_file = join(default['SETTINGS'], f)
 
     if f.endswith('.pbf'):
         osm_file = join(default['SETTINGS'], f)
 
-    if f.endswith('.poly'):
-        poly_file = join(default['SETTINGS'], f)
-
     """
     # Todo : need fix custom URL and sporadic diff : daily, hourly and minutely
     if f == 'custom_url_diff.txt':
@@ -81,44 +70,40 @@ for f in listdir(default['SETTINGS']):
             default['BASE_URL'] = content_file.read()
     """
 
-if not state_file:
-    print >> stderr, 'State file last.state.txt is missing in %s' % default['SETTINGS']
-    exit()
-
 if not osm_file:
     print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS']
     exit()
 
-if not poly_file:
+# In docker-compose, we should wait for the DB is ready.
-    print 'No *.poly detected in %s' % default['SETTINGS']
+print 'The checkup is OK. The container will continue soon, after the database.'
-else:
+sleep(45)
-    print '%s detected for clipping.' % poly_file
 
 # Finally launch the listening process.
 while True:
     # Check if diff to be imported is empty. If not, take the latest diff.
     diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
     if len(diff_to_be_imported):
-        timestamp = diff_to_be_imported[-1].split('.')[0]
+        file_name = diff_to_be_imported[-1].split('.')[0]
-        print "Timestamp from the latest not imported diff : %s" % timestamp
+        timestamp = file_name.split('->-')[1]
+        print 'Timestamp from the latest not imported diff : %s' % timestamp
     else:
         # Check if imported diff is empty. If not, take the latest diff.
         imported_diff = sorted(listdir(default['IMPORT_DONE']))
         if len(imported_diff):
-            print "Timestamp from the latest imported diff : %s" % timestamp
+            file_name = imported_diff[-1].split('.')[0]
-            timestamp = imported_diff[-1].split('.')[0]
+            timestamp = file_name.split('->-')[1]
+            print 'Timestamp from the latest imported diff : %s' % timestamp
 
         else:
             # Take the timestamp from original file.
-            state_file_settings = {}
+            command = ['osmconvert', osm_file, '--out-timestamp']
-            with open(state_file) as a_file:
+            processus = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-                for line in a_file:
+            timestamp, err = processus.communicate()
-                    if '=' in line:
-                        name, value = line.partition("=")[::2]
-                        state_file_settings[name] = value
 
-            timestamp = state_file_settings['timestamp'].strip()
+            # Remove new line
-            print "Timestamp from the original state file : %s" % timestamp
+            timestamp = timestamp.strip()
+
+            print 'Timestamp from the original state file : %s' % timestamp
 
     # Removing some \ in the timestamp.
     timestamp = timestamp.replace('\\', '')
@@ -129,13 +114,11 @@ while True:
     print 'Current time : %s' % current_time
 
     # Destination
-    file_name = '%s.osc.gz' % current_time
+    file_name = '%s->-%s.osc.gz' % (timestamp, current_time)
     file_path = join(default['IMPORT_QUEUE'], file_name)
 
     # Command
     command = ['osmupdate', '-v']
-    if poly_file:
-        command.append('-B=%s' % poly_file)
     command += ['--max-days=' + default['MAX_DAYS']]
     command += [default['DIFF']]
     command += ['--max-merge=' + default['MAX_MERGE']]

settings/mapping.json

@@ -1,4 +1,46 @@
 {
+    "generalized_tables": {
+        "waterareas_gen1": {
+            "source": "waterareas",
+            "sql_filter": "ST_Area(geometry)>50000.000000",
+            "tolerance": 50.0
+        },
+        "waterareas_gen0": {
+            "source": "waterareas_gen1",
+            "sql_filter": "ST_Area(geometry)>500000.000000",
+            "tolerance": 200.0
+        },
+        "roads_gen0": {
+            "source": "roads_gen1",
+            "sql_filter": null,
+            "tolerance": 200.0
+        },
+        "roads_gen1": {
+            "source": "roads",
+            "sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')",
+            "tolerance": 50.0
+        },
+        "waterways_gen0": {
+            "source": "waterways_gen1",
+            "sql_filter": null,
+            "tolerance": 200
+        },
+        "waterways_gen1": {
+            "source": "waterways",
+            "sql_filter": null,
+            "tolerance": 50.0
+        },
+        "landusages_gen1": {
+            "source": "landusages",
+            "sql_filter": "ST_Area(geometry)>50000.000000",
+            "tolerance": 50.0
+        },
+        "landusages_gen0": {
+            "source": "landusages_gen1",
+            "sql_filter": "ST_Area(geometry)>500000.000000",
+            "tolerance": 200.0
+        }
+    },
     "tables": {
         "landusages": {
             "fields": [
@@ -29,58 +71,57 @@
                 },
                 {
                     "args": {
-                        "ranks": [
+                        "values": [
-                            "pedestrian",
+                            "land",
-                            "footway",
-                            "playground",
-                            "park",
-                            "forest",
-                            "cemetery",
-                            "farmyard",
-                            "farm",
-                            "farmland",
-                            "wood",
-                            "meadow",
-                            "grass",
-                            "wetland",
-                            "village_green",
-                            "recreation_ground",
-                            "garden",
-                            "sports_centre",
-                            "pitch",
-                            "common",
-                            "allotments",
-                            "golf_course",
-                            "university",
-                            "school",
-                            "college",
-                            "library",
-                            "baracks",
-                            "fuel",
-                            "parking",
-                            "nature_reserve",
-                            "cinema",
-                            "theatre",
-                            "place_of_worship",
-                            "hospital",
-                            "scrub",
-                            "orchard",
-                            "vineyard",
-                            "zoo",
-                            "quarry",
-                            "residential",
-                            "retail",
-                            "commercial",
-                            "industrial",
-                            "railway",
-                            "heath",
                             "island",
-                            "land"
+                            "heath",
+                            "railway",
+                            "industrial",
+                            "commercial",
+                            "retail",
+                            "residential",
+                            "quarry",
+                            "zoo",
+                            "vineyard",
+                            "orchard",
+                            "scrub",
+                            "hospital",
+                            "place_of_worship",
+                            "theatre",
+                            "cinema",
+                            "nature_reserve",
+                            "parking",
+                            "fuel",
+                            "baracks",
+                            "library",
+                            "college",
+                            "school",
+                            "university",
+                            "golf_course",
+                            "allotments",
+                            "common",
+                            "pitch",
+                            "sports_centre",
+                            "garden",
+                            "recreation_ground",
+                            "village_green",
+                            "wetland",
+                            "grass",
+                            "meadow",
+                            "wood",
+                            "farmland",
+                            "farm",
+                            "farmyard",
+                            "cemetery",
+                            "forest",
+                            "park",
+                            "playground",
+                            "footway",
+                            "pedestrian"
                         ]
                     },
-                    "type": "zorder",
+                    "type": "enumerate",
-                    "name": "z_order",
+                    "name": "z_order"
-                    "key": "z_order"
                 }
             ],
             "type": "polygon",
@@ -116,7 +157,6 @@
                 ],
                 "natural": [
                     "wood",
-                    "water",
                     "land",
                     "scrub",
                     "wetland",
@@ -160,9 +200,6 @@
                 "highway": [
                     "pedestrian",
                     "footway"
-                ],
-                "waterway": [
-                    "riverbank"
                 ]
             }
         },
@@ -196,37 +233,6 @@
                 ]
             }
         },
-        "amenity_areas": {
-            "_comment": "for testing duplicate inserts with __any__ and exact match",
-            "fields": [
-                {
-                    "type": "id",
-                    "name": "osm_id",
-                    "key": null
-                },
-                {
-                    "type": "geometry",
-                    "name": "geometry",
-                    "key": null
-                },
-                {
-                    "type": "string",
-                    "name": "name",
-                    "key": "name"
-                },
-                {
-                    "type": "mapping_value",
-                    "name": "type",
-                    "key": null
-                }
-            ],
-            "type": "polygon",
-            "mapping": {
-                "building": [
-                    "shop"
-                ]
-            }
-        },
         "places": {
             "fields": [
                 {
@@ -251,22 +257,21 @@
                 },
                 {
                     "args": {
-                        "ranks": [
+                        "values": [
-                            "country",
+                            "locality",
-                            "state",
-                            "region",
-                            "county",
-                            "city",
-                            "town",
-                            "village",
-                            "hamlet",
                             "suburb",
-                            "locality"
+                            "hamlet",
+                            "village",
+                            "town",
+                            "city",
+                            "county",
+                            "region",
+                            "state",
+                            "country"
                         ]
                     },
-                    "type": "zorder",
+                    "type": "enumerate",
-                    "name": "z_order",
+                    "name": "z_order"
-                    "key": "z_order"
                 },
                 {
                     "type": "integer",
@@ -819,6 +824,54 @@
                     "__any__"
                 ]
             }
+        },
+        "waterareas": {
+            "fields": [
+                {
+                    "type": "id",
+                    "name": "osm_id",
+                    "key": null
+                },
+                {
+                    "type": "geometry",
+                    "name": "geometry",
+                    "key": null
+                },
+                {
+                    "type": "string",
+                    "name": "name",
+                    "key": "name"
+                },
+                {
+                    "type": "mapping_value",
+                    "name": "type",
+                    "key": null
+                },
+                {
+                    "type": "pseudoarea",
+                    "name": "area",
+                    "key": null
+                }
+            ],
+            "type": "polygon",
+            "mapping": {
+                "waterway": [
+                    "riverbank"
+                ],
+                "landuse": [
+                    "basin",
+                    "reservoir"
+                ],
+                "natural": [
+                    "water"
+                ],
+                "amenity": [
+                    "swimming_pool"
+                ],
+                "leisure": [
+                    "swimming_pool"
+                ]
+            }
         }
     }
 }

settings/timestamp.txt

@@ -0,0 +1 @@
+UNDEFINED