diff --git a/docker-compose.yml b/docker-compose.yml index 102e65a..fae04c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,14 +5,13 @@ volumes: import_done: import_queue: cache: - pg: - grafana: - pw2: + osm_settings: + pgadmin_data: services: db: # About the postgresql version, it should match in the dockerfile of docker-imposm3 - image: kartoza/postgis:11.0-2.5 + image: kartoza/postgis:12.0 hostname: db container_name: dockerosm_db environment: @@ -23,20 +22,34 @@ services: # - ALLOW_IP_RANGE= 0.0.0.0/0 volumes: - osm-postgis-data:/var/lib/postgresql - - ./settings:/home/settings # Uncomment to use the postgis database from outside the docker network # ports: # - "35432:5432" healthcheck: test: "exit 0" + osm_downloader: + image: kartoza/docker-osm:pbf-downloader + build: docker-osm-pbf + container_name: dockerosm_pbf_download + volumes: + # These are sharable to other containers + - osm_settings:/home/settings + environment: + # Read the README in docker-osm-pbf + - CONTINENT=africa + - COUNTRY=south-africa + - BASE_URL=http://download.geofabrik.de + - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings + - GEOJSON_URL='' + imposm: image: kartoza/docker-osm:imposm-latest build: docker-imposm3 container_name: dockerosm_imposm volumes: # These are sharable to other containers - - ./settings:/home/settings + - osm_settings:/home/settings - import_done:/home/import_done - import_queue:/home/import_queue - cache:/home/cache @@ -74,6 +87,7 @@ services: - QGIS_STYLE=yes # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB - CLIP=no + command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py" osmupdate: build: docker-osmupdate @@ -81,7 +95,7 @@ services: container_name: dockerosm_osmupdate volumes: # These are sharable to other containers - - ./settings:/home/settings + - osm_settings:/home/settings - import_done:/home/import_done - import_queue:/home/import_queue - cache:/home/cache @@ -108,30 +122,19 @@ services: # seconds between 2 executions of the script # if 0, then no update will be done, only the first initial import from the PBF - TIME=120 - pgwatch2: - image: cybertec/pgwatch2-postgres - hostname: pgwatch2 + command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py" + + pgadmin4: + image: dpage/pgadmin4:4.16 + hostname: pgadmin4 volumes: - - pg:/var/lib/postgresql - - grafana:/var/lib/grafana - - pw2:/pgwatch2/persistent-config + - pgadmin_data:/var/lib/pgadmin environment: - - PW2_TESTDB=false - - PW2_PG_SCHEMA_TYPE=metric-dbname-time - - PW2_WEBNOANONYMOUS=true - - PW2_WEBUSER=admin - - PW2_WEBPASSWORD=pgwatch2 - - PW2_WEBNOCOMPONENTLOGS=false - - PW2_WEBHOST=0.0.0.0 - - PW2_WEBPORT=8080 - - PW2_GRAFANANOANONYMOUS=true - - PW2_GRAFANAUSER=admin - - PW2_GRAFANAPASSWORD=pgwatch2 - - PW2_GRAFANA_BASEURL=http://0.0.0.0:3000 - restart: on-failure + - PGADMIN_DEFAULT_EMAIL=docker@gmail.com + - PGADMIN_DEFAULT_PASSWORD=docker ports: - - 3000:3000 - - 8080:8080 + - 6500:80 + restart: on-failure depends_on: db: - condition: service_healthy \ No newline at end of file + condition: service_healthy diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 9186e81..36bb627 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -303,12 +303,10 @@ class Importer(object): command += ['-srid', self.default['SRID']] command += ['-diffdir', self.default['SETTINGS']] command += ['-mapping', self.mapping_file] - command += ['-limitto', self.clip_json_file] command += ['-connection', self.postgis_uri] command += [join(self.default['IMPORT_QUEUE'], diff)] self.info(command.extend(args)) - if call(command) == 0: move( join(self.default['IMPORT_QUEUE'], diff), diff --git a/docker-osm-pbf/Dockerfile b/docker-osm-pbf/Dockerfile new file mode 100644 index 0000000..84d0e90 --- /dev/null +++ b/docker-osm-pbf/Dockerfile @@ -0,0 +1,14 @@ +FROM alpine:latest + +RUN apk --no-cache add --update bash + +ENV BASE_URL='http://download.geofabrik.de' +ENV CONTINENT='' +ENV COUNTRY='' +ENV MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings' +ENV GEOJSON_URL='' + +RUN mkdir /home/settings +ADD download.sh /download.sh + +ENTRYPOINT ["/bin/bash", "/download.sh"] diff --git a/docker-osm-pbf/README.md b/docker-osm-pbf/README.md new file mode 100644 index 0000000..4e3630d --- /dev/null +++ b/docker-osm-pbf/README.md @@ -0,0 +1,31 @@ +# Download Docker OSM Files + +This image is used to facilitate downloading of docker-osm files which are required to get the image +running. The image will download OSM PBF file, Mapping file, Clip Geojson and QGIS Style file. + +Environment variables + + +**BASE_URL='http://download.geofabrik.de'** + +This is used to download the OSM PBF file. Currently points to Geofabrik + +**CONTINENT=''** + +Used to specify what continent you need to download pbf from. This is mandatory eg `CONTINENT=africa` + +**COUNTRY=''** + +Used to specify which country you need to download pbf from. This is optional if you intent +to only use continent pbf. Eg `COUNTRY=lesotho` + +**MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings'** + +This currently points to the docker-osm repository to enable downloading of the mapping file, qgis_style + file. These files are mandatory in the running of docker-osm + +**GEOJSON_URL=''** + +This points to the geojson file that is used for clipping data in OSM. This can be empty if you do +not intent to use the clip functionality in docker-osm + diff --git a/docker-osm-pbf/download.sh b/docker-osm-pbf/download.sh new file mode 100644 index 0000000..f53bada --- /dev/null +++ b/docker-osm-pbf/download.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +CONTINENT_LOCKFILE=/home/settings/.${CONTINENT}_lock +COUNTRY_LOCKFILE=/home/settings/.${COUNTRY}_lock + +touch /home/settings/last.state.txt +touch /home/settings/timestamp.txt + +# Download OSM Mapping file and Associated data +if [ ! -f /home/settings/mapping.yml ]; then \ + wget -c ${MAPPING_URL}/mapping.yml -O /home/settings/mapping.yml +fi + +if [ ! -f /home/settings/qgis_style.sql ]; then \ + wget -c ${MAPPING_URL}/qgis_style.sql -O /home/settings/qgis_style.sql +fi +if [ ! -f /home/settings/post-pbf-import.sql ]; then \ + url=${MAPPING_URL}/post-pbf-import.sql + if curl --output /dev/null --silent --head --fail "${url}"; then + wget -c ${MAPPING_URL}/post-pbf-import.sql -O /home/settings/post-pbf-import.sql + else + echo "URL does not exist: ${url}" + fi +fi + +if [[ ! -f /home/settings/clip.geojson && -z ${GEOJSON_URL} ]]; then \ + echo "We are not downloading any Geojson" +else + wget -c ${GEOJSON_URL} -O /home/settings/clip.geojson +fi + +# Download OSM PBF +if [[ ! -f ${CONTINENT_LOCKFILE} && -z ${COUNTRY} ]]; then \ + + echo "${BASE_URL}/${CONTINENT}-latest.osm.pbf" + wget -c --no-check-certificate ${BASE_URL}/${CONTINENT}-latest.osm.pbf -O /tmp/${CONTINENT}.pbf + mv /tmp/$CONTINENT}.pbf /home/settings/country.pbf + touch ${CONTINENT_LOCKFILE} + + +elif [[ ! -f ${COUNTRY_LOCKFILE} ]]; then + + echo "${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf" + wget -c --no-check-certificate ${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf -O /tmp/${COUNTRY}.pbf + mv /tmp/${COUNTRY}.pbf /home/settings/country.pbf + touch ${COUNTRY_LOCKFILE} +fi + diff --git a/readme.md b/readme.md index ea4f7be..4d63eb2 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,9 @@ a smaller extent than the one specified by the PBF) and run the docker compose p ## Quick setup As a quick example, we are going to setup Docker-OSM with default values everywhere: -* Download a PBF file from http://download.geofabrik.de/ -* Put the file in the `settings` folder. +* Run the docker-compose file and make sure the environment variables are setup properly for +osm_downloader to download the correct pbf file. + * If you want to connect from your local QGIS Desktop: * In the file `docker-compose.yml`, uncomment the block: @@ -22,8 +23,9 @@ As a quick example, we are going to setup Docker-OSM with default values everywh ports: - "35432:5432" ``` -* Do `make run` in the build directory. This will download and execute the docker-osm project. It might be very long depending of your bandwidth and the PBF you are importing. -* In QGIS, add a new PostGIS connexion: `localhost`, database `gis`, port `35432`, `docker` for both username and password. +* Do `make run` in the build directory. This will download and execute the docker-osm project. +It might be very long depending of your bandwidth and the PBF you are importing. +* In QGIS, add a new PostGIS connection: `localhost`, database `gis`, port `35432`, `docker` for both username and password. * That's it! You have an OSM database, up and running. The update is done every 2 minutes from the main OSM website. For further reading and customizations, read below. @@ -47,15 +49,11 @@ your existing docker-compose project. In this example we will set up an OSM database for South Africa that will pull for updates every 2 minutes. -First get a PBF file from your area and put this file in the 'settings' folder. +Specify a PBF file for your area in the environment variables for `osm_downloader` container. You can download some PBF files on these URLS for instance : * http://download.geofabrik.de/ * http://download.openstreetmap.fr/extracts/ -```bash -cd settings -wget -c -O country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf -``` You must put only one PBF file in the settings folder. Only the last one will be read. @@ -122,7 +120,7 @@ Now build the docker images needed to run the application: ```bash docker-compose build -docker-compose up +docker-compose up ``` In production you should daemonize the services when bringing them up: