From 82fabf64db53a9469b176a0871785c14ec91b001 Mon Sep 17 00:00:00 2001 From: admire Date: Sun, 2 Feb 2020 15:20:29 +0200 Subject: [PATCH] Add lockfile to indicate importer has finished and run osmenrich afterwards --- .gitignore | 1 + docker-compose-rancher.yml | 162 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 20 +---- docker-imposm3/importer.py | 11 ++- readme.md | 15 +++- settings_downloader.sh | 51 ++++++++++++ 6 files changed, 240 insertions(+), 20 deletions(-) create mode 100644 docker-compose-rancher.yml create mode 100644 settings_downloader.sh diff --git a/.gitignore b/.gitignore index fb0a270..1649511 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ cache import_done import_queue settings/*.pbf +settings/*lock BACKUP-STYLES.sql settings/last.state.txt settings/timestamp.txt diff --git a/docker-compose-rancher.yml b/docker-compose-rancher.yml new file mode 100644 index 0000000..4837758 --- /dev/null +++ b/docker-compose-rancher.yml @@ -0,0 +1,162 @@ +version: '2.1' + +volumes: + osm-postgis-data: + import_done: + import_queue: + cache: + osm_settings: + pgadmin_data: + +services: + db: + # About the postgresql version, it should match in the dockerfile of docker-imposm3 + image: kartoza/postgis:12.0 + hostname: db + container_name: dockerosm_db + environment: + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - POSTGRES_DBNAME=gis + # Uncomment to expose the postgis database on the network + # - ALLOW_IP_RANGE= 0.0.0.0/0 + volumes: + - osm-postgis-data:/var/lib/postgresql + # Uncomment to use the postgis database from outside the docker network + # ports: + # - "35432:5432" + healthcheck: + test: "exit 0" + + osm_downloader: + image: kartoza/docker-osm:pbf-downloader + container_name: dockerosm_pbf_download + volumes: + # These are sharable to other containers + - osm_settings:/home/settings + environment: + # Read the README in docker-osm-pbf + - CONTINENT=africa + - COUNTRY=south-africa + - BASE_URL=http://download.geofabrik.de + - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings + - GEOJSON_URL='' + + imposm: + image: kartoza/docker-osm:imposm-latest + container_name: dockerosm_imposm + volumes: + # These are sharable to other containers + - osm_settings:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache + depends_on: + db: + condition: service_healthy + environment: + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + - POSTGRES_DBNAME=gis + - POSTGRES_PORT=5432 + - POSTGRES_HOST=db + # seconds between 2 executions of the script + # if 0, then no update will be done, only the first initial import from the PBF + - TIME=120 + # folder for settings (with *.json and *.sql) + - SETTINGS=settings + # folder for caching + - CACHE=cache + # folder for diff which has been imported + - IMPORT_DONE=import_done + # folder for diff which hasn't been imported yet + - IMPORT_QUEUE=import_queue + # it can be 3857 + - SRID=4326 + # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize + - OPTIMIZE=false + # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_PRODUCTION=public + # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_IMPORT=import + # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables + - DBSCHEMA_BACKUP=backup + # Install some styles if you are using the default mapping. It can be 'yes' or 'no' + - QGIS_STYLE=yes + # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB + - CLIP=no + command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py" + + osmupdate: + build: docker-osmupdate + image: kartoza/docker-osm:osmupdate-latest + container_name: dockerosm_osmupdate + volumes: + # These are sharable to other containers + - osm_settings:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache + depends_on: + db: + condition: service_healthy + environment: + # These are all currently the defaults but listed here for your + # convenience if you want to change them + # the maximum time range to assemble a cumulated changefile. + - MAX_DAYS=100 + # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. + - DIFF=sporadic + # argument to determine the maximum number of parallely processed changefiles. + - MAX_MERGE=7 + # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) + - COMPRESSION_LEVEL=1 + # change the URL to use a custom URL to fetch regional file updates. + - BASE_URL=http://planet.openstreetmap.org/replication/ + # folder for diff which hasn't been imported yet + - IMPORT_QUEUE=import_queue + # folder for diff which has been imported + - IMPORT_DONE=import_done + # seconds between 2 executions of the script + # if 0, then no update will be done, only the first initial import from the PBF + - TIME=120 + command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py" + + pgadmin4: + image: dpage/pgadmin4:4.16 + hostname: pgadmin4 + volumes: + - pgadmin_data:/var/lib/pgadmin + environment: + - PGADMIN_DEFAULT_EMAIL=docker@gmail.com + - PGADMIN_DEFAULT_PASSWORD=docker + ports: + - 6500:80 + restart: on-failure + depends_on: + db: + condition: service_healthy + + osmenrich: + build: docker-osmenrich + container_name: dockerosm_osmenrich + volumes: + # These are sharable to other containers + - ./settings:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache + depends_on: + db: + condition: service_healthy + environment: + # These are all currently the defaults but listed here for your + # convenience if you want to change them + # folder for diff which hasn't been imported yet + - IMPORT_QUEUE=import_queue + # folder for diff which has been imported + - IMPORT_DONE=import_done + # seconds between 2 executions of the script + # if 0, then no update will be done, only the first initial import from the PBF + - TIME=120 + command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index dd6060f..69214c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ volumes: import_done: import_queue: cache: - osm_settings: pgadmin_data: services: @@ -28,20 +27,6 @@ services: healthcheck: test: "exit 0" - osm_downloader: - image: kartoza/docker-osm:pbf-downloader - build: docker-osm-pbf - container_name: dockerosm_pbf_download - volumes: - # These are sharable to other containers - - osm_settings:/home/settings - environment: - # Read the README in docker-osm-pbf - - CONTINENT=africa - - COUNTRY=south-africa - - BASE_URL=http://download.geofabrik.de - - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings - - GEOJSON_URL='' imposm: image: kartoza/docker-osm:imposm-latest @@ -49,7 +34,7 @@ services: container_name: dockerosm_imposm volumes: # These are sharable to other containers - - osm_settings:/home/settings + - ./settings:/home/settings - import_done:/home/import_done - import_queue:/home/import_queue - cache:/home/cache @@ -95,7 +80,7 @@ services: container_name: dockerosm_osmupdate volumes: # These are sharable to other containers - - osm_settings:/home/settings + - ./settings:/home/settings - import_done:/home/import_done - import_queue:/home/import_queue - cache:/home/cache @@ -161,3 +146,4 @@ services: # seconds between 2 executions of the script # if 0, then no update will be done, only the first initial import from the PBF - TIME=120 + command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py" diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 88a08d8..31ba11c 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -20,7 +20,7 @@ """ from sys import exit, stderr -from os import environ, listdir +from os import environ, listdir, mknod from shutil import move from os.path import join, exists, abspath, isabs from psycopg2 import connect, OperationalError @@ -233,6 +233,14 @@ class Importer(object): # noinspection PyUnboundLocalVariable return self.cursor.fetchone()[0] + def lockfile(self): + setup_lockfile = join(self.default['SETTINGS'], 'importer.lock') + if not exists(setup_lockfile): + mknod(setup_lockfile) + else: + remove(setup_lockfile) + mknod(setup_lockfile) + def run(self): """First checker.""" osm_tables = self.locate_table('osm_%') @@ -278,6 +286,7 @@ class Importer(object): self.error(msg) else: self.info('Import PBF successful : %s' % self.osm_file) + self.lockfile() if self.post_import_file or self.qgis_style: # Set the password for psql diff --git a/readme.md b/readme.md index 4d63eb2..681812b 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,19 @@ a smaller extent than the one specified by the PBF) and run the docker compose p ## Quick setup As a quick example, we are going to setup Docker-OSM with default values everywhere: -* Run the docker-compose file and make sure the environment variables are setup properly for -osm_downloader to download the correct pbf file. +* Download a PBF file from http://download.geofabrik.de/ +* Put the file in the `settings` folder. + +Alternatively you can execute the `settings_downloader.sh` script to download the pbf and the clip file +`` +./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie +./settings_downloader.sh https://github.com/kartoza/docker-osm/blob/develop/settings/clip.geojson africa south-africa +`` + +If you use rancher for container management you can use the provided docker-compose.yml which allows you to automatically +download the settings onto the host machine by using environment variables. +* Run the docker-compose-rancher file and make sure the environment variables are setup properly for +osm_downloader to download the correct pbf file * If you want to connect from your local QGIS Desktop: * In the file `docker-compose.yml`, uncomment the block: diff --git a/settings_downloader.sh b/settings_downloader.sh new file mode 100644 index 0000000..a84a0dd --- /dev/null +++ b/settings_downloader.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + + + +GEOJSON_URL= + +if [ -n "$1" ] +then + GEOJSON_URL=$1 +fi + +CONTINENT=africa +if [ -n "$2" ] +then + CONTINENT=$2 +fi + +COUNTRY=south-africa + +if [ -n "$3" ] +then + COUNTRY=$3 +fi + +CONTINENT_LOCKFILE=./settings/.${CONTINENT}_lock +COUNTRY_LOCKFILE=./settings/.${COUNTRY}_lock +BASE_URL=http://download.geofabrik.de + +# Download OSM Mapping file and Geojson data + +if [[ ! -f ./settings/clip.geojson && -z ${GEOJSON_URL} ]]; then \ + echo "We are not downloading any Geojson" +else + wget -c ${GEOJSON_URL} -O ./settings/clip.geojson +fi + +# Download OSM PBF +if [[ ! -f ${CONTINENT_LOCKFILE} && -z ${COUNTRY} ]]; then \ + + echo "${BASE_URL}/${CONTINENT}-latest.osm.pbf" + wget -c --no-check-certificate ${BASE_URL}/${CONTINENT}-latest.osm.pbf -O ./settings/country.pbf + touch ${CONTINENT_LOCKFILE} + + +elif [[ ! -f ${COUNTRY_LOCKFILE} ]]; then + + echo "${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf" + wget -c --no-check-certificate ${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf -O ./settings/country.pbf + touch ${COUNTRY_LOCKFILE} +fi +