diff --git a/.gitignore b/.gitignore index d495830..fb0a270 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store .idea *.*~ *.log diff --git a/Makefile b/Makefile index f66df46..de00b0b 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,13 @@ rm: kill @echo "------------------------------------------------------------------" @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) rm +rm: clean + @echo + @echo "------------------------------------------------------------------" + @echo "Removing all volumes!!!! " + @echo "------------------------------------------------------------------" + @docker volume rm docker-osm_osm-postgis-data docker-osm_import_queue docker-osm_import_done docker-osm_cache + logs: @echo @echo "------------------------------------------------------------------" diff --git a/docker-compose-web.yml b/docker-compose-web.yml index 1eec2b5..57d395a 100644 --- a/docker-compose-web.yml +++ b/docker-compose-web.yml @@ -1,106 +1,10 @@ -storage: - image: ubuntu:latest - container_name: dockerosm_storage - hostname: storage - volumes: - # These are sharable to other containers - - ./settings:/home/settings - - /home/import_done - - /home/import_queue - - /home/cache - #- ./import_done:/home/import_done - #- ./import_queue:/home/import_queue - #- ./cache:/home/cache +# Useage: -db: - # About the postgresql version, it should match in the dockerfile of docker-imposm3 - image: kartoza/postgis:9.6-2.4 - container_name: dockerosm_db - hostname: db - environment: - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - - POSTGRES_DBNAME=gis - volumes_from: - - storage -# Uncomment to use the postgis database from outside the docker network -# ports: -# - "5432:5432" - -imposm: - # image: kartoza/docker-osm:imposm-latest - build: docker-imposm3 - container_name: dockerosm_imposm - volumes_from: - - storage - links: - - db:db - environment: - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - - POSTGRES_DBNAME=gis - - POSTGRES_PORT=5432 - - POSTGRES_HOST=db - # seconds between 2 executions of the script - # if 0, then no update will be done, only the first initial import from the PBF - - TIME=120 - # folder for settings (with *.json and *.sql) - - SETTINGS=settings - # folder for caching - - CACHE=cache - # folder for diff which has been imported - - IMPORT_DONE=import_done - # folder for diff which hasn't been imported yet - - IMPORT_QUEUE=import_queue - # it can be 3857 - - SRID=4326 - # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize - - OPTIMIZE=false - # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_PRODUCTION=public - # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_IMPORT=import - # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables - - DBSCHEMA_BACKUP=backup - # Install some styles if you are using the default mapping. It can be 'yes' or 'no' - - QGIS_STYLE=yes - # Use clip in the database - - CLIP=no - - -osmupdate: - # image: kartoza/docker-osm:osmupdate-latest - build: docker-osmupdate - container_name: dockerosm_osmupdate - volumes_from: - - storage - environment: - # These are all currently the defaults but listed here for your - # convenience if you want to change them - # the maximum time range to assemble a cumulated changefile. - - MAX_DAYS=100 - # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. - - DIFF=sporadic - # argument to determine the maximum number of parallely processed changefiles. - - MAX_MERGE=7 - # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) - - COMPRESSION_LEVEL=1 - # change the URL to use a custom URL to fetch regional file updates. - - BASE_URL=http://planet.openstreetmap.org/replication/ - # folder for diff which hasn't been imported yet - - IMPORT_QUEUE=import_queue - # folder for diff which has been imported - - IMPORT_DONE=import_done - # seconds between 2 executions of the script - # if 0, then no update will be done, only the first initial import from the PBF - - TIME=120 +# docker-compose -f docker-compose.yml -f docker-compose-web.yml qgisserver: image: kartoza/qgis-server:2.18 hostname: dockerosm_qgisserver - container_name: dockerosm_qgisserver - #volumes_from: - # - btsync volumes: - ./logs:/var/log/apache2 - ./web:/project @@ -109,16 +13,4 @@ qgisserver: - db:db ports: - 8198:80 - restart: on-failure:5 - -#btsync: -# image: kartoza/btsync -# container_name: dockerosm_btsync -# hostname: dockerosm_btsync -# volumes: -# - /web -# environment: -# This should be the read only key -# - SECRET=BBEBVOOYDPGO3GIIEQUMAIEJQNINOLX7V -# - DEVICE=readonly- -# restart: on-failure:5 + restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 07d7fba..250045f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,12 @@ version: '2.1' -services: - storage: - image: ubuntu:18.04 - volumes: - # These are sharable to other containers - - ./settings:/home/settings - - /home/import_done - - /home/import_queue - - /home/cache +volumes: + osm-postgis-data: + import_done: + import_queue: + cache: +services: db: # About the postgresql version, it should match in the dockerfile of docker-imposm3 image: kartoza/postgis:9.6-2.4 @@ -18,19 +15,24 @@ services: - POSTGRES_USER=docker - POSTGRES_PASS=docker - POSTGRES_DBNAME=gis - volumes_from: - - storage - # Uncomment to use the postgis database from outside the docker network - #ports: - #- "35432:5432" + volumes: + - 'osm-postgis-data:/var/lib/postgresql' + # Uncomment to use the postgis database from outside the docker network + ports: + - "35432:5432" healthcheck: test: "exit 0" + restart: unless-stopped imposm: - # image: kartoza/docker-osm:imposm-latest - build: docker-imposm3 - volumes_from: - - storage + image: kartoza/docker-osm:imposm-latest + #build: docker-imposm3 + volumes: + # These are sharable to other containers + - ./settings:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache depends_on: db: condition: service_healthy @@ -65,11 +67,17 @@ services: - QGIS_STYLE=yes # Use clip in the database - CLIP=no + restart: unless-stopped osmupdate: - build: docker-osmupdate - volumes_from: - - storage + #build: docker-osmupdate + image: kartoza/docker-osm:osmupdate-latest + volumes: + # These are sharable to other containers + - ./settings:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache environment: # These are all currently the defaults but listed here for your # convenience if you want to change them @@ -90,3 +98,4 @@ services: # seconds between 2 executions of the script # if 0, then no update will be done, only the first initial import from the PBF - TIME=120 + restart: unless-stopped diff --git a/readme.md b/readme.md index efa8b27..c18f9c9 100644 --- a/readme.md +++ b/readme.md @@ -1,18 +1,21 @@ # Docker-OSM -A docker compose project to setup an OSM PostGIS database with automatic updates from OSM periodically. +A docker compose project to setup an OSM PostGIS database with automatic +updates from OSM periodically. The only file you need is a PBF file and run the docker compose project. ## Docker cloud Dockerfiles are executed on https://cloud.docker.com + ```bash docker pull kartoza/docker-osm:imposm-latest docker pull kartoza/docker-osm:osmupdate-latest ``` -You need to copy/paste the docker-compose project and use the images hosted on the internet. -This is useful if you want to integrate Docker-OSM in your existing docker-compose project. +To run you can use the provided docker-compose project and use the images +hosted on the internet. This is useful if you want to integrate Docker-OSM in +your existing docker-compose project. ## Usage @@ -35,16 +38,19 @@ You must put only one PBF file in the settings folder. Only the last one will be ### OSM Features In `settings`, you can edit the `mapping.yml` to customize the PostGIS schema. -You can find the documentation about the mapping configuration on the imposm website: https://imposm.org/docs/imposm3/latest/mapping.html -The default file in Docker-OSM is coming from https://raw.githubusercontent.com/omniscale/imposm3/master/example-mapping.yml +You can find the documentation about the mapping configuration on the imposm +website: https://imposm.org/docs/imposm3/latest/mapping.html +The default file in Docker-OSM is coming from +https://raw.githubusercontent.com/omniscale/imposm3/master/example-mapping.yml ### Updates -You can configure the time interval in the docker-compose file. By default, it's two minutes. -If you set the TIME variable to 0, no diff files will be imported. +You can configure the time interval in the docker-compose file. By default, +it's two minutes. If you set the TIME variable to 0, no diff files will be +imported. -The default update stream is worldwide. -So even if you imported a local PBF, if you don't set a clipping area, you will end with data from all over the world. +The default update stream is worldwide. So even if you imported a local PBF, if +you don't set a clipping area, you will end with data from all over the world. ### Clipping @@ -58,8 +64,9 @@ You can remove the clip file : `make remove_clip`. ### QGIS Styles -The database is provided with some default styles. These styles will be loaded automatically when loaded in QGIS. -It's following the default OSM mapping from ImpOSM. +The database is provided with some default styles. These styles will be loaded +automatically when loaded in QGIS. It's following the default OSM mapping from +ImpOSM. ``` make import_styles @@ -87,9 +94,9 @@ docker-compose up -d ``` You can check the timestamp of your database by reading the file : -'settings/timestamp.txt' -or you can use : -'make timestamp' +``settings/timestamp.txt`` or you can use : + +``make timestamp`` ### Display @@ -110,8 +117,8 @@ Because in the docker-compose file, the link is made with the PostGIS database u ### Docker OSM Update -This docker image when run will fetch on a regular interval any new diff file -for all the changes that have happened in the world over the update interval. +This docker image, when run will regularly fetch any new diff file for all the +changes that have happened in the world over the update interval. You can also specify a custom url for fetching the diff if you wish to retrieve regional diffs rather than the global one. @@ -120,7 +127,7 @@ You can specify a polygonal area for the diff so that it will only apply feature from the diff that fall within that area. For example providing a polygon of the borders of Malawi will result in only Malawi features being extracted from the diff. -Note that the diff retrieved and options specified here are not related to the +**Note:** the diff retrieved and options specified here are not related to the initial base map used - so for example if your initial base map is for Malawi and you specify a diff area in Botswana, updated features in Botswana will be applied to your base map which only includes features from Malawi. For this reason, take @@ -140,6 +147,8 @@ http://download.openstreetmap.fr/extracts/africa/south_africa.state.txt ``docker run -v $('pwd')import-queue/:/home/import-queue -v $('pwd')base-pbf/:/home/base-pbf -v $('pwd')import-done/:/home/import-done -d osmupdate`` With -e, you can add some settings : + +``` - MAX_DAYS = 100, the maximum time range to assemble a cumulated changefile. - DIFF = sporadic, osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. - MAX_MERGE = 7, argument to determine the maximum number of parallely processed changefiles. @@ -149,6 +158,7 @@ With -e, you can add some settings : - IMPORT_DONE = import_done - OSM_PBF = osm_pbf - TIME = 120, seconds between two executions of the script +``` If you are using docker-compose, you can use these settings within the ```docker-compose.yml``` file. @@ -165,6 +175,8 @@ The container will look for an OSM file (*.pbf) and its state file (*.state.txt) in BASE_PBF. With -e, you can add some settings : + +``` - TIME = 120, seconds between 2 executions of the script - POSTGRES_USER = docker, default user - POSTGRES_PASS = docker, default password @@ -180,10 +192,19 @@ With -e, you can add some settings : - DBSCHEMA_PRODUCTION = public, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] - DBSCHEMA_IMPORT = import, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] - DBSCHEMA_BACKUP = backup, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] +``` You can adjust these preferences in the ```docker-compose.yml``` file provided in this repository. +# QGIS Server + +You can run a QGIS Server front end to the OSM mirroir by using the provided +docker-compose-web.yml file. For example: + +``` +docker-compose -f docker-compose.yml -f docker-compose-web.yml qgisserver up +``` # Credits