Download settings from a github repo (#89)

* Fix import error due to differences in geom type

* fix id missing in clip table and skipping importing shapefiles with geometry errors

* fix clip id error in the sql

* Change postgresql version from 9.6 to 11

* Added sql to validate geometry and other make commands to run them manually

* added pgwatch to docker-compose

* Modified importer.py to import clip file if it exists and changed logic for finding if tables are missing

* updated readme file

* fix logic on first run for importing data

* small changes to revert original logic

* remove clip functions since they are not needed because of the limitto function

* Added logic to check if the geojson should be used if it exist or not, added martin to server vector tiles from the DB

* update readme

* added recipe for downloading pbf,clip and mapping file from a given github repository

* removed pbf which was accidentally pushed

* fix volume name

* Added logc to check pbf download is finished

* use none standard ports

* fixed import error in imposm

* fix docker-compose and update descriptions

docker-compose.yml

@@ -5,14 +5,13 @@ volumes:
   import_done:
   import_queue:
   cache:
-  pg:
+  osm_settings:
-  grafana:
+  pgadmin_data:
-  pw2:
 
 services:
   db:
     # About the postgresql version, it should match in the dockerfile of docker-imposm3
-    image: kartoza/postgis:11.0-2.5
+    image: kartoza/postgis:12.0
     hostname: db
     container_name: dockerosm_db
     environment:
@@ -23,20 +22,34 @@ services:
       # - ALLOW_IP_RANGE= 0.0.0.0/0
     volumes:
       - osm-postgis-data:/var/lib/postgresql
-      - ./settings:/home/settings
     # Uncomment to use the postgis database from outside the docker network
     # ports:
     #  - "35432:5432"
     healthcheck:
         test: "exit 0"
 
+  osm_downloader:
+    image: kartoza/docker-osm:pbf-downloader
+    build: docker-osm-pbf
+    container_name: dockerosm_pbf_download
+    volumes:
+      # These are sharable to other containers
+      - osm_settings:/home/settings
+    environment:
+      # Read the README in docker-osm-pbf
+      - CONTINENT=africa
+      - COUNTRY=south-africa
+      - BASE_URL=http://download.geofabrik.de
+      - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings
+      - GEOJSON_URL=''
+
   imposm:
     image: kartoza/docker-osm:imposm-latest
     build: docker-imposm3
     container_name: dockerosm_imposm
     volumes:
       # These are sharable to other containers
-      - ./settings:/home/settings
+      - osm_settings:/home/settings
       - import_done:/home/import_done
       - import_queue:/home/import_queue
       - cache:/home/cache
@@ -74,6 +87,7 @@ services:
       - QGIS_STYLE=yes
       # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
       - CLIP=no
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
 
   osmupdate:
     build: docker-osmupdate
@@ -81,7 +95,7 @@ services:
     container_name: dockerosm_osmupdate
     volumes:
       # These are sharable to other containers
-      - ./settings:/home/settings
+      - osm_settings:/home/settings
       - import_done:/home/import_done
       - import_queue:/home/import_queue
       - cache:/home/cache
@@ -108,30 +122,19 @@ services:
       # seconds between 2 executions of the script
       # if 0, then no update will be done, only the first initial import from the PBF
       - TIME=120
-  pgwatch2:
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
-    image: cybertec/pgwatch2-postgres
+
-    hostname: pgwatch2
+  pgadmin4:
+    image: dpage/pgadmin4:4.16
+    hostname: pgadmin4
     volumes:
-      - pg:/var/lib/postgresql
+      - pgadmin_data:/var/lib/pgadmin
-      - grafana:/var/lib/grafana
-      - pw2:/pgwatch2/persistent-config
     environment:
-      - PW2_TESTDB=false
+      - PGADMIN_DEFAULT_EMAIL=docker@gmail.com
-      - PW2_PG_SCHEMA_TYPE=metric-dbname-time
+      - PGADMIN_DEFAULT_PASSWORD=docker
-      - PW2_WEBNOANONYMOUS=true
-      - PW2_WEBUSER=admin
-      - PW2_WEBPASSWORD=pgwatch2
-      - PW2_WEBNOCOMPONENTLOGS=false
-      - PW2_WEBHOST=0.0.0.0
-      - PW2_WEBPORT=8080
-      - PW2_GRAFANANOANONYMOUS=true
-      - PW2_GRAFANAUSER=admin
-      - PW2_GRAFANAPASSWORD=pgwatch2
-      - PW2_GRAFANA_BASEURL=http://0.0.0.0:3000
-    restart: on-failure
     ports:
-      - 3000:3000
+      - 6500:80
-      - 8080:8080
+    restart: on-failure
     depends_on:
       db:
-        condition: service_healthy
+        condition: service_healthy

docker-imposm3/importer.py

@@ -303,12 +303,10 @@ class Importer(object):
                     command += ['-srid', self.default['SRID']]
                     command += ['-diffdir', self.default['SETTINGS']]
                     command += ['-mapping', self.mapping_file]
-                    command += ['-limitto', self.clip_json_file]
                     command += ['-connection', self.postgis_uri]
                     command += [join(self.default['IMPORT_QUEUE'], diff)]
 
                     self.info(command.extend(args))
-
                     if call(command) == 0:
                         move(
                             join(self.default['IMPORT_QUEUE'], diff),

docker-osm-pbf/Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:latest
+
+RUN apk --no-cache add --update bash
+
+ENV BASE_URL='http://download.geofabrik.de'
+ENV CONTINENT=''
+ENV COUNTRY=''
+ENV MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings'
+ENV GEOJSON_URL=''
+
+RUN mkdir /home/settings
+ADD download.sh /download.sh
+
+ENTRYPOINT ["/bin/bash", "/download.sh"]

docker-osm-pbf/README.md

@@ -0,0 +1,31 @@
+# Download Docker OSM Files
+
+This image is used to facilitate downloading of docker-osm files which are required to get the image 
+running. The image will download OSM PBF file, Mapping file, Clip Geojson and QGIS Style file.
+
+Environment variables
+
+
+**BASE_URL='http://download.geofabrik.de'** 
+
+This is used to download the OSM PBF file. Currently points to Geofabrik
+
+**CONTINENT=''**
+
+Used to specify what continent you need to download pbf from. This is mandatory eg `CONTINENT=africa`
+
+**COUNTRY=''** 
+
+Used to specify which country you need to download pbf from. This is optional if you intent
+to only use continent pbf. Eg `COUNTRY=lesotho`
+
+**MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings'**
+  
+This currently points to the docker-osm repository to enable downloading of the mapping file, qgis_style
+ file. These files are mandatory in the running of docker-osm
+
+**GEOJSON_URL=''** 
+
+This points to the geojson file that is used for clipping data in OSM. This can be empty if you do 
+not intent to use the clip functionality in docker-osm
+

docker-osm-pbf/download.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+CONTINENT_LOCKFILE=/home/settings/.${CONTINENT}_lock
+COUNTRY_LOCKFILE=/home/settings/.${COUNTRY}_lock
+
+touch /home/settings/last.state.txt
+touch /home/settings/timestamp.txt
+
+# Download OSM Mapping file and Associated data
+if [ ! -f /home/settings/mapping.yml ]; then \
+  wget -c ${MAPPING_URL}/mapping.yml -O /home/settings/mapping.yml
+fi
+
+if [ ! -f /home/settings/qgis_style.sql ]; then \
+  wget -c ${MAPPING_URL}/qgis_style.sql -O /home/settings/qgis_style.sql
+fi
+if [ ! -f /home/settings/post-pbf-import.sql ]; then \
+  url=${MAPPING_URL}/post-pbf-import.sql
+  if curl --output /dev/null --silent --head --fail "${url}"; then
+    wget -c ${MAPPING_URL}/post-pbf-import.sql -O /home/settings/post-pbf-import.sql
+  else
+    echo "URL does not exist: ${url}"
+  fi
+fi
+
+if [[ ! -f /home/settings/clip.geojson && -z ${GEOJSON_URL} ]]; then \
+  echo "We are not downloading any Geojson"
+else
+  wget -c ${GEOJSON_URL} -O /home/settings/clip.geojson
+fi
+
+# Download OSM PBF
+if [[ ! -f ${CONTINENT_LOCKFILE} && -z ${COUNTRY} ]]; then \
+
+  echo "${BASE_URL}/${CONTINENT}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}-latest.osm.pbf -O /tmp/${CONTINENT}.pbf
+  mv /tmp/$CONTINENT}.pbf /home/settings/country.pbf
+  touch ${CONTINENT_LOCKFILE}
+
+
+elif [[ ! -f ${COUNTRY_LOCKFILE} ]]; then
+
+  echo "${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf -O /tmp/${COUNTRY}.pbf
+  mv /tmp/${COUNTRY}.pbf /home/settings/country.pbf
+  touch ${COUNTRY_LOCKFILE}
+fi
+

readme.md

@@ -12,8 +12,9 @@ a smaller extent than the one specified by the PBF) and run the docker compose p
 ## Quick setup
 
 As a quick example, we are going to setup Docker-OSM with default values everywhere:
-* Download a PBF file from http://download.geofabrik.de/
+* Run the docker-compose file and make sure the environment variables are setup properly for
-* Put the file in the `settings` folder.
+osm_downloader to download the correct pbf file.
+
 * If you want to connect from your local QGIS Desktop:
   * In the file `docker-compose.yml`, uncomment the block:
 
@@ -22,8 +23,9 @@ As a quick example, we are going to setup Docker-OSM with default values everywh
 ports:
  - "35432:5432"
 ```
-* Do `make run` in the build directory. This will download and execute the docker-osm project. It might be very long depending of your bandwidth and the PBF you are importing.
+* Do `make run` in the build directory. This will download and execute the docker-osm project. 
-* In QGIS, add a new PostGIS connexion: `localhost`, database `gis`, port `35432`, `docker` for both username and password.
+It might be very long depending of your bandwidth and the PBF you are importing.
+* In QGIS, add a new PostGIS connection: `localhost`, database `gis`, port `35432`, `docker` for both username and password.
 * That's it! You have an OSM database, up and running. The update is done every 2 minutes from the main OSM website.
 
 For further reading and customizations, read below.
@@ -47,15 +49,11 @@ your existing docker-compose project.
 In this example we will set up an OSM database for South Africa that 
 will pull for updates every 2 minutes.
 
-First get a PBF file from your area and put this file in the 'settings' folder.
+Specify a PBF file for your area in the environment variables for `osm_downloader` container.
 You can download some PBF files on these URLS for instance :
 * http://download.geofabrik.de/
 * http://download.openstreetmap.fr/extracts/
 
-```bash
-cd settings
-wget -c -O country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf
-```
 
 You must put only one PBF file in the settings folder. Only the last one will be read.
 
@@ -122,7 +120,7 @@ Now build the docker images needed to run the application:
 
 ```bash
 docker-compose build
-docker-compose up
+docker-compose up 
 ```
 
 In production you should daemonize the services when bringing them up: