Download settings from a github repo (#89)

* Fix import error due to differences in geom type * fix id missing in clip table and skipping importing shapefiles with geometry errors * fix clip id error in the sql * Change postgresql version from 9.6 to 11 * Added sql to validate geometry and other make commands to run them manually * added pgwatch to docker-compose * Modified importer.py to import clip file if it exists and changed logic for finding if tables are missing * updated readme file * fix logic on first run for importing data * small changes to revert original logic * remove clip functions since they are not needed because of the limitto function * Added logic to check if the geojson should be used if it exist or not, added martin to server vector tiles from the DB * update readme * added recipe for downloading pbf,clip and mapping file from a given github repository * removed pbf which was accidentally pushed * fix volume name * Added logc to check pbf download is finished * use none standard ports * fixed import error in imposm * fix docker-compose and update descriptions
2020-01-03 13:44:39 +02:00 · 2020-01-03 13:44:39 +02:00 · 6828c0b1e8
commit 6828c0b1e8
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -5,14 +5,13 @@ volumes:
  import_done:
  import_queue:
  cache:
-  pg:
-  grafana:
-  pw2:
+  osm_settings:
+  pgadmin_data:

 services:
  db:
    # About the postgresql version, it should match in the dockerfile of docker-imposm3
-    image: kartoza/postgis:11.0-2.5
+    image: kartoza/postgis:12.0
    hostname: db
    container_name: dockerosm_db
    environment:
@ -23,20 +22,34 @@ services:
      # - ALLOW_IP_RANGE= 0.0.0.0/0
    volumes:
      - osm-postgis-data:/var/lib/postgresql
-      - ./settings:/home/settings
    # Uncomment to use the postgis database from outside the docker network
    # ports:
    #  - "35432:5432"
    healthcheck:
        test: "exit 0"

+  osm_downloader:
+    image: kartoza/docker-osm:pbf-downloader
+    build: docker-osm-pbf
+    container_name: dockerosm_pbf_download
+    volumes:
+      # These are sharable to other containers
+      - osm_settings:/home/settings
+    environment:
+      # Read the README in docker-osm-pbf
+      - CONTINENT=africa
+      - COUNTRY=south-africa
+      - BASE_URL=http://download.geofabrik.de
+      - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings
+      - GEOJSON_URL=''
+
  imposm:
    image: kartoza/docker-osm:imposm-latest
    build: docker-imposm3
    container_name: dockerosm_imposm
    volumes:
      # These are sharable to other containers
-      - ./settings:/home/settings
+      - osm_settings:/home/settings
      - import_done:/home/import_done
      - import_queue:/home/import_queue
      - cache:/home/cache
@ -74,6 +87,7 @@ services:
      - QGIS_STYLE=yes
      # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
      - CLIP=no
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"

  osmupdate:
    build: docker-osmupdate
@ -81,7 +95,7 @@ services:
    container_name: dockerosm_osmupdate
    volumes:
      # These are sharable to other containers
-      - ./settings:/home/settings
+      - osm_settings:/home/settings
      - import_done:/home/import_done
      - import_queue:/home/import_queue
      - cache:/home/cache
@ -108,30 +122,19 @@ services:
      # seconds between 2 executions of the script
      # if 0, then no update will be done, only the first initial import from the PBF
      - TIME=120
-  pgwatch2:
-    image: cybertec/pgwatch2-postgres
-    hostname: pgwatch2
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
+
+  pgadmin4:
+    image: dpage/pgadmin4:4.16
+    hostname: pgadmin4
    volumes:
-      - pg:/var/lib/postgresql
-      - grafana:/var/lib/grafana
-      - pw2:/pgwatch2/persistent-config
+      - pgadmin_data:/var/lib/pgadmin
    environment:
-      - PW2_TESTDB=false
-      - PW2_PG_SCHEMA_TYPE=metric-dbname-time
-      - PW2_WEBNOANONYMOUS=true
-      - PW2_WEBUSER=admin
-      - PW2_WEBPASSWORD=pgwatch2
-      - PW2_WEBNOCOMPONENTLOGS=false
-      - PW2_WEBHOST=0.0.0.0
-      - PW2_WEBPORT=8080
-      - PW2_GRAFANANOANONYMOUS=true
-      - PW2_GRAFANAUSER=admin
-      - PW2_GRAFANAPASSWORD=pgwatch2
-      - PW2_GRAFANA_BASEURL=http://0.0.0.0:3000
-    restart: on-failure
+      - PGADMIN_DEFAULT_EMAIL=docker@gmail.com
+      - PGADMIN_DEFAULT_PASSWORD=docker
    ports:
-      - 3000:3000
-      - 8080:8080
+      - 6500:80
+    restart: on-failure
    depends_on:
      db:
-        condition: service_healthy
+        condition: service_healthy
--- a/docker-imposm3/importer.py
+++ b/docker-imposm3/importer.py
@ -303,12 +303,10 @@ class Importer(object):
                    command += ['-srid', self.default['SRID']]
                    command += ['-diffdir', self.default['SETTINGS']]
                    command += ['-mapping', self.mapping_file]
-                    command += ['-limitto', self.clip_json_file]
                    command += ['-connection', self.postgis_uri]
                    command += [join(self.default['IMPORT_QUEUE'], diff)]

                    self.info(command.extend(args))
-
                    if call(command) == 0:
                        move(
                            join(self.default['IMPORT_QUEUE'], diff),
--- a/docker-osm-pbf/Dockerfile
+++ b/docker-osm-pbf/Dockerfile
@ -0,0 +1,14 @@
+FROM alpine:latest
+
+RUN apk --no-cache add --update bash
+
+ENV BASE_URL='http://download.geofabrik.de'
+ENV CONTINENT=''
+ENV COUNTRY=''
+ENV MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings'
+ENV GEOJSON_URL=''
+
+RUN mkdir /home/settings
+ADD download.sh /download.sh
+
+ENTRYPOINT ["/bin/bash", "/download.sh"]
--- a/docker-osm-pbf/README.md
+++ b/docker-osm-pbf/README.md
@ -0,0 +1,31 @@
+# Download Docker OSM Files
+
+This image is used to facilitate downloading of docker-osm files which are required to get the image 
+running. The image will download OSM PBF file, Mapping file, Clip Geojson and QGIS Style file.
+
+Environment variables
+
+
+**BASE_URL='http://download.geofabrik.de'** 
+
+This is used to download the OSM PBF file. Currently points to Geofabrik
+
+**CONTINENT=''**
+
+Used to specify what continent you need to download pbf from. This is mandatory eg `CONTINENT=africa`
+
+**COUNTRY=''** 
+
+Used to specify which country you need to download pbf from. This is optional if you intent
+to only use continent pbf. Eg `COUNTRY=lesotho`
+
+**MAPPING_URL='https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings'**
+  
+This currently points to the docker-osm repository to enable downloading of the mapping file, qgis_style
+ file. These files are mandatory in the running of docker-osm
+
+**GEOJSON_URL=''** 
+
+This points to the geojson file that is used for clipping data in OSM. This can be empty if you do 
+not intent to use the clip functionality in docker-osm
+
--- a/docker-osm-pbf/download.sh
+++ b/docker-osm-pbf/download.sh
@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+CONTINENT_LOCKFILE=/home/settings/.${CONTINENT}_lock
+COUNTRY_LOCKFILE=/home/settings/.${COUNTRY}_lock
+
+touch /home/settings/last.state.txt
+touch /home/settings/timestamp.txt
+
+# Download OSM Mapping file and Associated data
+if [ ! -f /home/settings/mapping.yml ]; then \
+  wget -c ${MAPPING_URL}/mapping.yml -O /home/settings/mapping.yml
+fi
+
+if [ ! -f /home/settings/qgis_style.sql ]; then \
+  wget -c ${MAPPING_URL}/qgis_style.sql -O /home/settings/qgis_style.sql
+fi
+if [ ! -f /home/settings/post-pbf-import.sql ]; then \
+  url=${MAPPING_URL}/post-pbf-import.sql
+  if curl --output /dev/null --silent --head --fail "${url}"; then
+    wget -c ${MAPPING_URL}/post-pbf-import.sql -O /home/settings/post-pbf-import.sql
+  else
+    echo "URL does not exist: ${url}"
+  fi
+fi
+
+if [[ ! -f /home/settings/clip.geojson && -z ${GEOJSON_URL} ]]; then \
+  echo "We are not downloading any Geojson"
+else
+  wget -c ${GEOJSON_URL} -O /home/settings/clip.geojson
+fi
+
+# Download OSM PBF
+if [[ ! -f ${CONTINENT_LOCKFILE} && -z ${COUNTRY} ]]; then \
+
+  echo "${BASE_URL}/${CONTINENT}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}-latest.osm.pbf -O /tmp/${CONTINENT}.pbf
+  mv /tmp/$CONTINENT}.pbf /home/settings/country.pbf
+  touch ${CONTINENT_LOCKFILE}
+
+
+elif [[ ! -f ${COUNTRY_LOCKFILE} ]]; then
+
+  echo "${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf -O /tmp/${COUNTRY}.pbf
+  mv /tmp/${COUNTRY}.pbf /home/settings/country.pbf
+  touch ${COUNTRY_LOCKFILE}
+fi
+
--- a/readme.md
+++ b/readme.md
@ -12,8 +12,9 @@ a smaller extent than the one specified by the PBF) and run the docker compose p
 ## Quick setup

 As a quick example, we are going to setup Docker-OSM with default values everywhere:
-* Download a PBF file from http://download.geofabrik.de/
-* Put the file in the `settings` folder.
+* Run the docker-compose file and make sure the environment variables are setup properly for
+osm_downloader to download the correct pbf file.
+
 * If you want to connect from your local QGIS Desktop:
  * In the file `docker-compose.yml`, uncomment the block:

@ -22,8 +23,9 @@ As a quick example, we are going to setup Docker-OSM with default values everywh
 ports:
 - "35432:5432"
 ```
-* Do `make run` in the build directory. This will download and execute the docker-osm project. It might be very long depending of your bandwidth and the PBF you are importing.
-* In QGIS, add a new PostGIS connexion: `localhost`, database `gis`, port `35432`, `docker` for both username and password.
+* Do `make run` in the build directory. This will download and execute the docker-osm project. 
+It might be very long depending of your bandwidth and the PBF you are importing.
+* In QGIS, add a new PostGIS connection: `localhost`, database `gis`, port `35432`, `docker` for both username and password.
 * That's it! You have an OSM database, up and running. The update is done every 2 minutes from the main OSM website.

 For further reading and customizations, read below.
@ -47,15 +49,11 @@ your existing docker-compose project.
 In this example we will set up an OSM database for South Africa that 
 will pull for updates every 2 minutes.

-First get a PBF file from your area and put this file in the 'settings' folder.
+Specify a PBF file for your area in the environment variables for `osm_downloader` container.
 You can download some PBF files on these URLS for instance :
 * http://download.geofabrik.de/
 * http://download.openstreetmap.fr/extracts/

-```bash
-cd settings
-wget -c -O country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf
-```

 You must put only one PBF file in the settings folder. Only the last one will be read.

@ -122,7 +120,7 @@ Now build the docker images needed to run the application:

 ```bash
 docker-compose build
-docker-compose up
+docker-compose up 
 ```

 In production you should daemonize the services when bringing them up: