Merge pull request #11 from Gustry/develop

Merge osm_pbf into settings and update doc

.gitignore

@@ -1,6 +1,7 @@
 .idea
 *.*~
-osm_pbf/*.pbf
+settings/*.pbf
-osm_pbf/*.state.txt
+settings/*.state.txt
-osm_pbf/*.poly
+settings/*.poly
 settings/last.state.txt
+settings/custom_url_diff.txt

docker-compose.yml

@@ -3,7 +3,6 @@ storage:
   hostname: storage
   volumes:
     # These are sharable to other containers
-    - ./osm_pbf:/home/osm_pbf
     - ./settings:/home/settings
     - /home/import_done
     - /home/import_queue

docker-imposm3/importer.py

@@ -41,7 +41,6 @@ default = {
     'PORT': '5432',
     'SETTINGS': 'settings',
     'CACHE': 'cache',
-    'OSM_PBF': 'osm_pbf',
     'IMPORT_DONE': 'import_done',
     'IMPORT_QUEUE': 'import_queue',
     'SRID': '4326',
@@ -52,7 +51,6 @@ default = {
 }
 
 # Check if we overwrite default values.
-# env = [var.lower() for var in environ]
 for key in default.keys():
     if key in environ:
         default[key] = environ[key]
@@ -82,7 +80,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % (
     default['DATABASE'])
 
 # Check folders.
-folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'OSM_PBF', 'SETTINGS', 'CACHE']
+folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE']
 for folder in folders:
     if not isabs(default[folder]):
         # Get the absolute path.
@@ -98,28 +96,27 @@ state_file = None
 osm_file = None
 mapping_file = None
 post_import_file = None
-for f in listdir(default['OSM_PBF']):
+for f in listdir(default['SETTINGS']):
     if f.endswith('.state.txt'):
-        state_file = join(default['OSM_PBF'], f)
+        state_file = join(default['SETTINGS'], f)
 
     if f.endswith('.pbf'):
-        osm_file = join(default['OSM_PBF'], f)
+        osm_file = join(default['SETTINGS'], f)
 
-if not osm_file:
-    print >> stderr, 'OSM file *.pbf is missing in %s' % default['OSM_PBF']
-    exit()
-
-if not state_file:
-    print >> stderr, 'State file *.state.txt is missing in %s' % default['OSM_PBF']
-    exit()
-
-for f in listdir(default['SETTINGS']):
     if f.endswith('.json'):
         mapping_file = join(default['SETTINGS'], f)
 
     if f.endswith('.sql'):
         post_import_file = join(default['SETTINGS'], f)
 
+if not osm_file:
+    print >> stderr, 'OSM file *.pbf is missing in %s' % default['SETTINGS']
+    exit()
+
+if not state_file:
+    print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
+    exit()
+
 if not mapping_file:
     print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS']
     exit()

docker-osmupdate/download.py

@@ -19,7 +19,7 @@
  ***************************************************************************/
 """
 
-from os.path import exists, join, isabs, abspath
+from os.path import exists, join, isabs, abspath, isfile
 from os import listdir, environ
 from sys import exit
 from subprocess import call
@@ -39,7 +39,7 @@ default = {
     'BASE_URL': 'http://planet.openstreetmap.org/replication/',
     'IMPORT_QUEUE': 'import_queue',
     'IMPORT_DONE': 'import_done',
-    'OSM_PBF': 'osm_pbf',
+    'SETTINGS': 'settings',
     'TIME': 120,
 }
 
@@ -48,7 +48,7 @@ for key in default.keys():
         default[key] = environ[key]
 
 # Folders
-folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'OSM_PBF']
+folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS']
 for folder in folders:
     if not isabs(default[folder]):
         # Get the absolute path.
@@ -63,26 +63,33 @@ for folder in folders:
 state_file = None
 osm_file = None
 poly_file = None
-for f in listdir(default['OSM_PBF']):
+for f in listdir(default['SETTINGS']):
     if f.endswith('.state.txt'):
-        state_file = join(default['OSM_PBF'], f)
+        state_file = join(default['SETTINGS'], f)
 
     if f.endswith('.pbf'):
-        osm_file = join(default['OSM_PBF'], f)
+        osm_file = join(default['SETTINGS'], f)
 
     if f.endswith('.poly'):
-        poly_file = join(default['OSM_PBF'], f)
+        poly_file = join(default['SETTINGS'], f)
+
+    """
+    # Todo : need fix custom URL and sporadic diff : daily, hourly and minutely
+    if f == 'custom_url_diff.txt':
+        with open(join(default['SETTINGS'], f), 'r') as content_file:
+            default['BASE_URL'] = content_file.read()
+    """
 
 if not state_file:
-    print >> stderr, 'State file *.state.txt is missing in %s' % default['OSM_PBF']
+    print >> stderr, 'State file *.state.txt is missing in %s' % default['SETTINGS']
     exit()
 
 if not osm_file:
-    print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['OSM_PBF']
+    print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['SETTINGS']
     exit()
 
 if not poly_file:
-    print 'No *.poly detected in %s' % default['OSM_PBF']
+    print 'No *.poly detected in %s' % default['SETTINGS']
 else:
     print '%s detected for clipping.' % poly_file
 

pbf_downloader.py

@@ -37,21 +37,25 @@ if url:
     state = diff + 'state.txt'
     print 'Polygon file : ' + poly_file
     print 'PBF file : ' + pbf_file
-    print 'Diff (not used): ' + diff
+    print 'Diff URL : ' + diff
     print 'state : ' + state
 
     print 'Downloading PBF'
-    commands = ['wget', '-c', '-O', 'osm_pbf/country.pbf', pbf_file]
+    commands = ['wget', '-c', '-O', 'settings/country.pbf', pbf_file]
     call(commands)
 
     print 'Downloading polygon'
-    commands = ['wget', '-c', '-O', 'osm_pbf/country.poly', poly_file]
+    commands = ['wget', '-c', '-O', 'settings/country.poly', poly_file]
     call(commands)
 
     print 'Downloading state'
-    commands = ['wget', '-c', '-O', 'osm_pbf/country.state.txt', state]
+    commands = ['wget', '-c', '-O', 'settings/country.state.txt', state]
     call(commands)
 
+    print 'Setting custom URL diff'
+    with open('settings/custom_url_diff.txt', 'w') as f:
+        f.write(diff)
+
 else:
     print 'This area is unkown in geofabrik or in our script. Check with the list argument.'
 

readme.md

@@ -2,7 +2,7 @@
 ## Usage
 
 In this example we will set up an OSM database for South Africa that 
-will poll for updates every hour.
+will pull for updates every 2 minutes.
 
 First fetch the latest South Africa osm binary dump file (.pbf) and state file.
 I will write the example as generically as possible so that you can substitute
@@ -15,6 +15,15 @@ wget -c -O country.state.txt http://download.openstreetmap.fr/extracts/africa/so
 wget -c -O polygon.poly http://download.geofabrik.de/africa/south-africa-and-lesotho.poly
 ```
 
+or you can use the PBF downloader using Geofabrik.
+To get the list of available countries in Geofabrik :
+``python pbf_downloader.py list``
+
+You can download a country or a continent :
+``python pbf_downloader.py south-africa-and-lesotho``
+
+The script will download the PBF file, the state file and the polygon for clipping.
+
 ```
 docker-compose build
 docker-compose up

settings/readme.md

@@ -1,7 +0,0 @@
-This folder should contain a json file for the mapping. You can also put a
-SQL file, it will be executed after the PBF import.
-
-Optionally you can include a .poly file (see http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Format)
-in this directory, which will be used as the clip extents for diffs. The
-base name of the .poly file is not important - the first .poly file encountered
-in this folder will be used.