diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a1891f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +*.*~ +osm_pbf/*.pbf +osm_pbf/*.state.txt diff --git a/base-pbf/readme.md b/base-pbf/readme.md deleted file mode 100644 index fca8662..0000000 --- a/base-pbf/readme.md +++ /dev/null @@ -1 +0,0 @@ -This folder should contain a *.pbf and *.state.txt. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 51b3ca6..e33efc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,16 @@ storage: - build: osm-storage + image: ubuntu:latest hostname: storage volumes: - - ../docker-osmupdate/base_pbf:/home/base_pbf + # These are sharable to other containers + - ./osm_pbf:/home/osm_pbf - ./settings:/home/settings + - /home/import_done + - /home/import_queue + - /home/cache db: - image: etrimaille/postgis - #image: kartoza/postgis - #build: ../docker-postgis + image: kartoza/postgis hostname: db environment: - USERNAME=docker @@ -16,8 +18,8 @@ db: # - HSTORE=true imposm: - image: etrimaille/imposm - #build: ../docker-imposm3 + #image: kartoza/imposm + build: docker-imposm3 volumes_from: - storage links: @@ -28,12 +30,14 @@ imposm: - PORT=5432 - HOST=db - DATEBASE=gis + # in seconds - TIME=120 osmupdate: - image: etrimaille/osmupdate - #build: ../docker-osmupdate + #image: etrimaille/osmupdate + build: docker-osmupdate volumes_from: - storage environment: - - TIME=120 \ No newline at end of file + # in seconds + - TIME=120 diff --git a/docker-imposm3/.idea/.name b/docker-imposm3/.idea/.name deleted file mode 100644 index 994d59b..0000000 --- a/docker-imposm3/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -docker-imposm3 \ No newline at end of file diff --git a/docker-imposm3/.idea/docker-imposm3.iml b/docker-imposm3/.idea/docker-imposm3.iml deleted file mode 100644 index d0876a7..0000000 --- a/docker-imposm3/.idea/docker-imposm3.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/encodings.xml b/docker-imposm3/.idea/encodings.xml deleted file mode 100644 index d821048..0000000 --- a/docker-imposm3/.idea/encodings.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml b/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 9def44c..0000000 --- a/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml b/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 3b31283..0000000 --- a/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/misc.xml b/docker-imposm3/.idea/misc.xml deleted file mode 100644 index 7b4d854..0000000 --- a/docker-imposm3/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/modules.xml b/docker-imposm3/.idea/modules.xml deleted file mode 100644 index cce79ba..0000000 --- a/docker-imposm3/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/scopes/scope_settings.xml b/docker-imposm3/.idea/scopes/scope_settings.xml deleted file mode 100644 index 922003b..0000000 --- a/docker-imposm3/.idea/scopes/scope_settings.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/vcs.xml b/docker-imposm3/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/docker-imposm3/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/docker-imposm3/.idea/workspace.xml b/docker-imposm3/.idea/workspace.xml deleted file mode 100644 index 03fd26b..0000000 --- a/docker-imposm3/.idea/workspace.xml +++ /dev/null @@ -1,857 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - 1437474153749 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docker-imposm3/README.md b/docker-imposm3/README.md index ee837f4..f9cec8d 100644 --- a/docker-imposm3/README.md +++ b/docker-imposm3/README.md @@ -1,6 +1,13 @@ ## Docker Imposm -The container will look for an OSM file (*.pbf) and its state file (*.state.txt) in BASE_PBF. +This image will take care of doing the initial load for the selected region +(e.g. planet, or a country such as Malawi) into your database. It will then +apply at a regulart interval (default is 2 minutes) any diff that arrives +in the /home/import_queue folder to the postgis OSM database. The diffs +are fetched by a separate container (see osm_update container). + +The container will look for an OSM file (*.pbf) and its state file +(*.state.txt) in BASE_PBF. With -e, you can add some settings : - TIME = 120, seconds between 2 executions of the script @@ -17,4 +24,4 @@ With -e, you can add some settings : - OPTIMIZE = false, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#optimize] - DBSCHEMA_PRODUCTION = public, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] - DBSCHEMA_IMPORT = import, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] - - DBSCHEMA_BACKUP = backup, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] \ No newline at end of file + - DBSCHEMA_BACKUP = backup, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 559d7e4..a84c0c4 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -10,6 +10,7 @@ from sys import stderr # In docker-compose, we should wait the DB is ready. sleep(30) +# All these default values can be overwritten by env vars default = { 'TIME': 120, 'USER': 'docker', @@ -19,7 +20,7 @@ default = { 'PORT': '5432', 'SETTINGS': 'settings', 'CACHE': 'cache', - 'BASE_PBF': 'base_pbf', + 'OSM_PBF': 'osm_pbf', 'IMPORT_DONE': 'import_done', 'IMPORT_QUEUE': 'import_queue', 'SRID': '4326', @@ -60,7 +61,7 @@ postgis_uri = 'postgis://%s:%s@%s/%s' % ( default['DATABASE']) # Check folders. -folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'BASE_PBF', 'SETTINGS', 'CACHE'] +folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'OSM_PBF', 'SETTINGS', 'CACHE'] for folder in folders: if not isabs(default[folder]): # Get the absolute path. @@ -76,19 +77,19 @@ state_file = None osm_file = None mapping_file = None post_import_file = None -for f in listdir(default['BASE_PBF']): +for f in listdir(default['OSM_PBF']): if f.endswith('.state.txt'): - state_file = join(default['BASE_PBF'], f) + state_file = join(default['OSM_PBF'], f) if f.endswith('.osm.pbf'): - osm_file = join(default['BASE_PBF'], f) + osm_file = join(default['OSM_PBF'], f) if not osm_file: - print >> stderr, 'OSM file *.pbf is missing in %s' % default['BASE_PBF'] + print >> stderr, 'OSM file *.pbf is missing in %s' % default['OSM_PBF'] exit() if not state_file: - print >> stderr, 'State file *.state.txt is missing in %s' % default['BASE_PBF'] + print >> stderr, 'State file *.state.txt is missing in %s' % default['OSM_PBF'] exit() for f in listdir(default['SETTINGS']): diff --git a/docker-osm-storage/Dockerfile b/docker-osm-storage/Dockerfile deleted file mode 100644 index c4f8c76..0000000 --- a/docker-osm-storage/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:latest -MAINTAINER Etienne Trimaille - -WORKDIR /home -RUN mkdir settings import_done import_queue base_pbf cache -VOLUME /home/settings /home/import_done /home/import_queue /home/base_pbf /home/cache - -CMD True \ No newline at end of file diff --git a/docker-osmupdate/.idea/.name b/docker-osmupdate/.idea/.name deleted file mode 100644 index 076c77d..0000000 --- a/docker-osmupdate/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -docker-osmupdate \ No newline at end of file diff --git a/docker-osmupdate/.idea/docker-osmupdate.iml b/docker-osmupdate/.idea/docker-osmupdate.iml deleted file mode 100644 index 1be76f1..0000000 --- a/docker-osmupdate/.idea/docker-osmupdate.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/encodings.xml b/docker-osmupdate/.idea/encodings.xml deleted file mode 100644 index d821048..0000000 --- a/docker-osmupdate/.idea/encodings.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml b/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 86a287d..0000000 --- a/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml b/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 3b31283..0000000 --- a/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/misc.xml b/docker-osmupdate/.idea/misc.xml deleted file mode 100644 index 7b4d854..0000000 --- a/docker-osmupdate/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/modules.xml b/docker-osmupdate/.idea/modules.xml deleted file mode 100644 index b9e5349..0000000 --- a/docker-osmupdate/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/scopes/scope_settings.xml b/docker-osmupdate/.idea/scopes/scope_settings.xml deleted file mode 100644 index 922003b..0000000 --- a/docker-osmupdate/.idea/scopes/scope_settings.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/vcs.xml b/docker-osmupdate/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/docker-osmupdate/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/docker-osmupdate/.idea/workspace.xml b/docker-osmupdate/.idea/workspace.xml deleted file mode 100644 index c3bb32f..0000000 --- a/docker-osmupdate/.idea/workspace.xml +++ /dev/null @@ -1,682 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - 1437385466775 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docker-osmupdate/readme.md b/docker-osmupdate/readme.md index 83bb263..34b50c3 100644 --- a/docker-osmupdate/readme.md +++ b/docker-osmupdate/readme.md @@ -1,6 +1,27 @@ ## Docker OSM Update -You should have 3 folders : base_pbf, import_queue, import_done +This docker image when run will fetch on a regular interval any new diff file +for all the changes that have happened in the world over the update interval. + +You can also specify a custom url for fetching the diff if you wish to retrieve +regional diffs rather than the global one. + +You can specify a polygonal area for the diff so that it will only apply features +from the diff that fall within that area. For example providing a polygon of the +borders of Malawi will result in only Malawi features being extracted from the diff. + +Note that the diff retrieved and options specified here are not related to the +initial base map used - so for example if your initial base map is for Malawi and +you specify a diff area in Botswana, updated features in Botswana will be applied +to your base map which only includes features from Malawi. For this reason, take +care to ensure that your diff area coincides with the region covered by your +original base map. + +Once the diff has been downloaded, it is placed into /home/import_queue where +it will be picked up by the long running imposm3 container, which will apply +the diff to the database. + +You should have 3 folders : osm_pbf, import_queue, import_done Put a state file in base-pbf like this one : http://download.openstreetmap.fr/extracts/africa/south_africa.state.txt @@ -16,5 +37,5 @@ With -e, you can add some settings : - BASE_URL = http://planet.openstreetmap.org/replication/, change the URL to use a custom URL to fetch regional file updates. - IMPORT_QUEUE = import_queue - IMPORT_DONE = import_done - - BASE_PBF = base_pbf - - TIME = 120, secondes between two executions of the script \ No newline at end of file + - OSM_PBF = osm_pbf + - TIME = 120, secondes between two executions of the script diff --git a/osm_pbf/readme.md b/osm_pbf/readme.md new file mode 100644 index 0000000..a6eec6f --- /dev/null +++ b/osm_pbf/readme.md @@ -0,0 +1,4 @@ +This folder should contain a *.pbf and *.state.txt. + +The first pbf file encoutered will be imported, for this reason we recommend +to only place one pbf in this folder so you can be sure which will be imported. diff --git a/readme.md b/readme.md index 75afebd..24a0260 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,32 @@ -Copy an OSM file (*.pbf) and its state file(*.state.txt) into base-pbf. The state file contains the timestamp of the OSM file. You can add a [polygon file](http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Format) for the clipping. +Copy an OSM file (*.pbf) and its state file(*.state.txt) into base-pbf. +The state file contains the timestamp of the OSM file. You can add a +[polygon file](http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Format) +for the clipping. Check that a *.json is present in 'settings' for the mapping. ``docker-compose build`` ``docker-compose up`` -You should read the documentation about [docker-imposm](https://github.com/gustry/docker-imposm3) and [docker-osmupdate](https://github.com/Gustry/docker-osmupdate) for settings. +You should read the documentation about [docker-imposm] +(https://github.com/gustry/docker-imposm3) and +[docker-osmupdate](https://github.com/Gustry/docker-osmupdate) for +settings. + + +# Example usage + +In this example we will set up an OSM database for South Africa that +will poll for updates every hour. + +First fetch the latest South Africa osm binary dump file (.pbf) and state file. +I will write the example as generically as possible so that you can substitute +your own country or region here. + +``` +mkdir osm +cd osm +wget -c -O country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf +wget -c http://download.openstreetmap.fr/extracts/africa/south_africa.state.txt +``` + + diff --git a/settings/readme.md b/settings/readme.md index 239b046..92ac40a 100644 --- a/settings/readme.md +++ b/settings/readme.md @@ -1 +1,7 @@ -This folder should contain a json file for the mapping. You can also put a SQL file, it will be executed after the PBF import. \ No newline at end of file +This folder should contain a json file for the mapping. You can also put a +SQL file, it will be executed after the PBF import. + +Optionally you can include a .poly file (see http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Format) +in this directory, which will be used as the clip extents for diffs. The +base name of the .poly file is not important - the first .poly file encountered +in this folder will be used.