fix some logic in osmupdate

2021-02-13 21:40:31 +02:00 · 2021-02-13 21:40:31 +02:00 · 2c6d0f67b0
commit 2c6d0f67b0
--- a/.env
+++ b/.env
@ -36,7 +36,7 @@ DBSCHEMA_BACKUP=backup
 # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
 QGIS_STYLE=yes
 # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
-CLIP=no
+CLIP=yes
 # These are all currently the defaults but listed here for your
 # convenience if you want to change them
 # the maximum time range to assemble a cumulated changefile.
--- a/docker-compose-rancher.yml
+++ b/docker-compose-rancher.yml
@ -1,162 +0,0 @@
-version: '2.1'
-
-volumes:
-  osm-postgis-data:
-  import_done:
-  import_queue:
-  cache:
-  osm_settings:
-  pgadmin_data:
-
-services:
-  db:
-    # About the postgresql version, it should match in the dockerfile of docker-imposm3
-    image: kartoza/postgis:12.0
-    hostname: db
-    container_name: dockerosm_db
-    environment:
-      - POSTGRES_USER=docker
-      - POSTGRES_PASS=docker
-      - POSTGRES_DBNAME=gis
-      # Uncomment to expose the postgis database on the network
-      # - ALLOW_IP_RANGE= 0.0.0.0/0
-    volumes:
-      - osm-postgis-data:/var/lib/postgresql
-    # Uncomment to use the postgis database from outside the docker network
-    # ports:
-    #  - "35432:5432"
-    healthcheck:
-      test: "exit 0"
-
-  osm_downloader:
-    image: kartoza/docker-osm:pbf-downloader
-    container_name: dockerosm_pbf_download
-    volumes:
-      # These are sharable to other containers
-      - osm_settings:/home/settings
-    environment:
-      # Read the README in docker-osm-pbf
-      - CONTINENT=africa
-      - COUNTRY=south-africa
-      - BASE_URL=http://download.geofabrik.de
-      - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings
-      - GEOJSON_URL=''
-
-  imposm:
-    image: kartoza/docker-osm:imposm-latest
-    container_name: dockerosm_imposm
-    volumes:
-      # These are sharable to other containers
-      - osm_settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      - POSTGRES_USER=docker
-      - POSTGRES_PASS=docker
-      - POSTGRES_DBNAME=gis
-      - POSTGRES_PORT=5432
-      - POSTGRES_HOST=db
-      # seconds between 2 executions of the script
-      # if 0, then no update will be done, only the first initial import from the PBF
-      - TIME=120
-      # folder for settings (with *.json and *.sql)
-      - SETTINGS=settings
-      # folder for caching
-      - CACHE=cache
-      # folder for diff which has been imported
-      - IMPORT_DONE=import_done
-      # folder for diff which hasn't been imported yet
-      - IMPORT_QUEUE=import_queue
-      # it can be 3857
-      - SRID=4326
-      # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
-      - OPTIMIZE=false
-      # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_PRODUCTION=public
-      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_IMPORT=import
-      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
-      - DBSCHEMA_BACKUP=backup
-      # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
-      - QGIS_STYLE=yes
-      # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
-      - CLIP=no
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
-
-  osmupdate:
-    build: docker-osmupdate
-    image: kartoza/docker-osm:osmupdate-latest
-    container_name: dockerosm_osmupdate
-    volumes:
-      # These are sharable to other containers
-      - osm_settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      # These are all currently the defaults but listed here for your
-      # convenience if you want to change them
-      # the maximum time range to assemble a cumulated changefile.
-      - MAX_DAYS=100
-      # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
-      - DIFF=sporadic
-      # argument to determine the maximum number of parallely processed changefiles.
-      - MAX_MERGE=7
-      # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
-      - COMPRESSION_LEVEL=1
-      # change the URL to use a custom URL to fetch regional file updates.
-      - BASE_URL=http://planet.openstreetmap.org/replication/
-      # folder for diff which hasn't been imported yet
-      - IMPORT_QUEUE=import_queue
-      # folder for diff which has been imported
-      - IMPORT_DONE=import_done
-      # seconds between 2 executions of the script
-      # if 0, then no update will be done, only the first initial import from the PBF
-      - TIME=120
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
-
-  pgadmin4:
-    image: dpage/pgadmin4:4.16
-    hostname: pgadmin4
-    volumes:
-      - pgadmin_data:/var/lib/pgadmin
-    environment:
-      - PGADMIN_DEFAULT_EMAIL=docker@gmail.com
-      - PGADMIN_DEFAULT_PASSWORD=docker
-    ports:
-      - 6500:80
-    restart: on-failure
-    depends_on:
-      db:
-        condition: service_healthy
-
-  osmenrich:
-    build: docker-osmenrich
-    container_name: dockerosm_osmenrich
-    volumes:
-      # These are sharable to other containers
-      - ./settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      # These are all currently the defaults but listed here for your
-      # convenience if you want to change them
-      # folder for diff which hasn't been imported yet
-      - IMPORT_QUEUE=import_queue
-      # folder for diff which has been imported
-      - IMPORT_DONE=import_done
-      # seconds between 2 executions of the script
-      # if 0, then no update will be done, only the first initial import from the PBF
-      - TIME=120
-    command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py"
--- a/docker-compose-web.yml
+++ b/docker-compose-web.yml
@ -1,4 +1,4 @@
-version: '3.9'
+version: '2.1'

 volumes:
  osm-postgis-data:
@ -34,7 +34,8 @@ services:
      - import_queue:/home/import_queue
      - cache:/home/cache
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASS=${POSTGRES_PASS}
@ -62,7 +63,8 @@ services:
    volumes_from:
      - imposm
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - MAX_DAYS=${MAX_DAYS}
      - DIFF=${DIFF}
@ -86,14 +88,16 @@ services:
      - ${PGDB_PORT}:80
    restart: on-failure
    depends_on:
-      - db
+      db:
+        condition: service_healthy

  osmenrich:
    build: docker-osmenrich
    volumes_from:
      - imposm
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - IMPORT_QUEUE=${IMPORT_QUEUE}
      - IMPORT_DONE=${IMPORT_DONE}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,4 +1,4 @@
-version: '3.9'
+version: '2.1'

 volumes:
  osm-postgis-data:
@ -35,7 +35,8 @@ services:
      - import_queue:/home/import_queue
      - cache:/home/cache
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASS=${POSTGRES_PASS}
@ -63,7 +64,8 @@ services:
    volumes_from:
      - imposm
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - MAX_DAYS=${MAX_DAYS}
      - DIFF=${DIFF}
@ -73,10 +75,9 @@ services:
      - IMPORT_QUEUE=${IMPORT_QUEUE}
      - IMPORT_DONE=${IMPORT_DONE}
      - TIME=${TIME}
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"

  pgadmin4:
-    image: dpage/pgadmin4:4.16
+    image: dpage/pgadmin4:latest
    hostname: pgadmin4
    volumes:
      - pgadmin_data:/var/lib/pgadmin
@ -87,7 +88,8 @@ services:
      - ${PGDB_PORT}:80
    restart: on-failure
    depends_on:
-      - db
+      db:
+        condition: service_healthy

  osmenrich:
    build: docker-osmenrich
@ -95,7 +97,8 @@ services:
    volumes_from:
      - imposm
    depends_on:
-      - db
+      db:
+        condition: service_healthy
    environment:
      - IMPORT_QUEUE=${IMPORT_QUEUE}
      - IMPORT_DONE=${IMPORT_DONE}
--- a/docker-osmupdate/download.py
+++ b/docker-osmupdate/download.py
@ -19,11 +19,11 @@
 ***************************************************************************/
 """

-from os.path import exists, join, isabs, abspath
-from os import listdir, environ
-from sys import exit, stderr
-from subprocess import call, Popen, PIPE
 from datetime import datetime
+from os import listdir, environ
+from os.path import exists, join, isabs, abspath
+from subprocess import call, Popen, PIPE
+from sys import exit, stderr
 from time import sleep


@ -82,12 +82,8 @@ class Downloader(object):

            if f.endswith('.pbf'):
                self.osm_file = join(self.default['SETTINGS'], f)
-
-        if not self.osm_file:
-            msg = 'OSM file *.osm.pbf is missing in %s' % self.default['SETTINGS']
-            self.error(msg)
-        else:
-            self.info('OSM PBF file: ' + self.osm_file)
+        while not exists(self.osm_file):
+            sleep(float(self.default['TIME']))

        self.info('The checkup is OK.')

--- a/readme.md
+++ b/readme.md
@ -21,11 +21,6 @@ Alternatively you can execute the `settings_downloader.sh` script to download th
 ./settings_downloader.sh https://github.com/kartoza/docker-osm/raw/develop/settings/clip.geojson africa south-africa
 ```

-If you use rancher for container management you can use the provided docker-compose.yml which allows you to automatically
-download the settings onto the host machine by using environment variables.
-* Run the docker-compose-rancher file and make sure the environment variables are setup properly for
-osm_downloader to download the correct pbf file
-
 * If you want to connect from your local QGIS Desktop:
  * In the file `docker-compose.yml`, uncomment the block: