From 006cc467ab75b5ff3f42fffa8ee3cd9cc1367bba Mon Sep 17 00:00:00 2001 From: Etienne Trimaille Date: Thu, 30 Jul 2015 11:38:13 +0200 Subject: [PATCH] merge docker-osmupdate docker-imposm --- base-pbf/readme.md | 1 + docker-imposm3/.idea/.name | 1 + docker-imposm3/.idea/docker-imposm3.iml | 8 + docker-imposm3/.idea/encodings.xml | 4 + .../inspectionProfiles/Project_Default.xml | 20 + .../inspectionProfiles/profiles_settings.xml | 7 + docker-imposm3/.idea/misc.xml | 4 + docker-imposm3/.idea/modules.xml | 8 + .../.idea/scopes/scope_settings.xml | 5 + docker-imposm3/.idea/vcs.xml | 6 + docker-imposm3/.idea/workspace.xml | 857 +++++++++ docker-imposm3/Dockerfile | 24 + docker-imposm3/README.md | 20 + docker-imposm3/importer.py | 165 ++ docker-imposm3/requirements.txt | 2 + .../Dockerfile | 0 docker-osmupdate/.idea/.name | 1 + docker-osmupdate/.idea/docker-osmupdate.iml | 8 + docker-osmupdate/.idea/encodings.xml | 4 + .../inspectionProfiles/Project_Default.xml | 13 + .../inspectionProfiles/profiles_settings.xml | 7 + docker-osmupdate/.idea/misc.xml | 4 + docker-osmupdate/.idea/modules.xml | 8 + .../.idea/scopes/scope_settings.xml | 5 + docker-osmupdate/.idea/vcs.xml | 6 + docker-osmupdate/.idea/workspace.xml | 682 +++++++ docker-osmupdate/71-apt-cacher-ng | 2 + docker-osmupdate/Dockerfile | 29 + docker-osmupdate/download.py | 122 ++ docker-osmupdate/osmupdate.c | 1650 +++++++++++++++++ docker-osmupdate/readme.md | 20 + settings/post-pbf-import.sql | 0 32 files changed, 3693 insertions(+) create mode 100644 base-pbf/readme.md create mode 100644 docker-imposm3/.idea/.name create mode 100644 docker-imposm3/.idea/docker-imposm3.iml create mode 100644 docker-imposm3/.idea/encodings.xml create mode 100644 docker-imposm3/.idea/inspectionProfiles/Project_Default.xml create mode 100644 docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 docker-imposm3/.idea/misc.xml create mode 100644 docker-imposm3/.idea/modules.xml create mode 100644 docker-imposm3/.idea/scopes/scope_settings.xml create mode 100644 docker-imposm3/.idea/vcs.xml create mode 100644 docker-imposm3/.idea/workspace.xml create mode 100644 docker-imposm3/Dockerfile create mode 100644 docker-imposm3/README.md create mode 100644 docker-imposm3/importer.py create mode 100644 docker-imposm3/requirements.txt rename {osm-storage => docker-osm-storage}/Dockerfile (100%) create mode 100644 docker-osmupdate/.idea/.name create mode 100644 docker-osmupdate/.idea/docker-osmupdate.iml create mode 100644 docker-osmupdate/.idea/encodings.xml create mode 100644 docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml create mode 100644 docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 docker-osmupdate/.idea/misc.xml create mode 100644 docker-osmupdate/.idea/modules.xml create mode 100644 docker-osmupdate/.idea/scopes/scope_settings.xml create mode 100644 docker-osmupdate/.idea/vcs.xml create mode 100644 docker-osmupdate/.idea/workspace.xml create mode 100644 docker-osmupdate/71-apt-cacher-ng create mode 100644 docker-osmupdate/Dockerfile create mode 100644 docker-osmupdate/download.py create mode 100644 docker-osmupdate/osmupdate.c create mode 100644 docker-osmupdate/readme.md create mode 100644 settings/post-pbf-import.sql diff --git a/base-pbf/readme.md b/base-pbf/readme.md new file mode 100644 index 0000000..fca8662 --- /dev/null +++ b/base-pbf/readme.md @@ -0,0 +1 @@ +This folder should contain a *.pbf and *.state.txt. \ No newline at end of file diff --git a/docker-imposm3/.idea/.name b/docker-imposm3/.idea/.name new file mode 100644 index 0000000..994d59b --- /dev/null +++ b/docker-imposm3/.idea/.name @@ -0,0 +1 @@ +docker-imposm3 \ No newline at end of file diff --git a/docker-imposm3/.idea/docker-imposm3.iml b/docker-imposm3/.idea/docker-imposm3.iml new file mode 100644 index 0000000..d0876a7 --- /dev/null +++ b/docker-imposm3/.idea/docker-imposm3.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/encodings.xml b/docker-imposm3/.idea/encodings.xml new file mode 100644 index 0000000..d821048 --- /dev/null +++ b/docker-imposm3/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml b/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..9def44c --- /dev/null +++ b/docker-imposm3/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,20 @@ + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml b/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..3b31283 --- /dev/null +++ b/docker-imposm3/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/misc.xml b/docker-imposm3/.idea/misc.xml new file mode 100644 index 0000000..7b4d854 --- /dev/null +++ b/docker-imposm3/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/modules.xml b/docker-imposm3/.idea/modules.xml new file mode 100644 index 0000000..cce79ba --- /dev/null +++ b/docker-imposm3/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/scopes/scope_settings.xml b/docker-imposm3/.idea/scopes/scope_settings.xml new file mode 100644 index 0000000..922003b --- /dev/null +++ b/docker-imposm3/.idea/scopes/scope_settings.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/vcs.xml b/docker-imposm3/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/docker-imposm3/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/docker-imposm3/.idea/workspace.xml b/docker-imposm3/.idea/workspace.xml new file mode 100644 index 0000000..03fd26b --- /dev/null +++ b/docker-imposm3/.idea/workspace.xml @@ -0,0 +1,857 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + 1437474153749 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker-imposm3/Dockerfile b/docker-imposm3/Dockerfile new file mode 100644 index 0000000..fd6be18 --- /dev/null +++ b/docker-imposm3/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.4 +MAINTAINER Etienne Trimaille + +RUN apt-get update && apt-get install -y python-pip \ + libprotobuf-dev libleveldb-dev libgeos-dev \ + libpq-dev python-dev \ + --no-install-recommends + +RUN ln -s /usr/lib/libgeos_c.so /usr/lib/libgeos.so + +VOLUME ['/home/base_pbf', '/home/import_done', '/home/import_queue', '/home/cache] + +WORKDIR $GOPATH +RUN go get github.com/tools/godep +RUN git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/imposm3 +RUN cd src/github.com/omniscale/imposm3 && godep go install ./... + +ADD requirements.txt /home/requirements.txt +RUN pip install -r /home/requirements.txt + +ADD importer.py /home/ + +WORKDIR /home +CMD ["python", "-u", "/home/importer.py"] diff --git a/docker-imposm3/README.md b/docker-imposm3/README.md new file mode 100644 index 0000000..ee837f4 --- /dev/null +++ b/docker-imposm3/README.md @@ -0,0 +1,20 @@ +## Docker Imposm + +The container will look for an OSM file (*.pbf) and its state file (*.state.txt) in BASE_PBF. + +With -e, you can add some settings : + - TIME = 120, seconds between 2 executions of the script + - USER = docker, default user + - PASSWORD = docker, default password + - HOST = db + - PORT = 5432 + - SETTINGS = settings, folder for settings (with *.json and *.sql) + - CACHE = cache, folder for caching + - BASE_PBF = base_pbf, folder the OSM file + - IMPORT_DONE = import_done, folder for diff which has been imported + - IMPORT_QUEUE = import_queue, folder for diff which hasn't been imported yet + - SRID = 4326, it can be 3857 + - OPTIMIZE = false, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#optimize] + - DBSCHEMA_PRODUCTION = public, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] + - DBSCHEMA_IMPORT = import, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] + - DBSCHEMA_BACKUP = backup, check (Imposm)[http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables] \ No newline at end of file diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py new file mode 100644 index 0000000..559d7e4 --- /dev/null +++ b/docker-imposm3/importer.py @@ -0,0 +1,165 @@ +from sys import exit +from os import environ, listdir +from shutil import move +from os.path import join, exists, abspath, isabs +from psycopg2 import connect, OperationalError +from subprocess import call +from time import sleep +from sys import stderr + +# In docker-compose, we should wait the DB is ready. +sleep(30) + +default = { + 'TIME': 120, + 'USER': 'docker', + 'PASSWORD': 'docker', + 'DATABASE': 'gis', + 'HOST': 'db', + 'PORT': '5432', + 'SETTINGS': 'settings', + 'CACHE': 'cache', + 'BASE_PBF': 'base_pbf', + 'IMPORT_DONE': 'import_done', + 'IMPORT_QUEUE': 'import_queue', + 'SRID': '4326', + 'OPTIMIZE': 'false', + 'DBSCHEMA_PRODUCTION': 'public', + 'DBSCHEMA_IMPORT': 'import', + 'DBSCHEMA_BACKUP': 'backup' +} + +# Check if we overwrite default values. +# env = [var.lower() for var in environ] +for key in default.keys(): + if key in environ: + default[key] = environ[key] + +# Check valid SRID. +if default['SRID'] not in ['4326', '3857']: + print >> stderr, 'SRID not supported : %s' % default['srid'] + exit() + +# Check postgis. +try: + connection = connect( + "dbname='%s' user='%s' host='%s' password='%s'" % ( + default['DATABASE'], + default['USER'], + default['HOST'], + default['PASSWORD'])) + cursor = connection.cursor() +except OperationalError as e: + print >> stderr, e + exit() + +postgis_uri = 'postgis://%s:%s@%s/%s' % ( + default['USER'], + default['PASSWORD'], + default['HOST'], + default['DATABASE']) + +# Check folders. +folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'BASE_PBF', 'SETTINGS', 'CACHE'] +for folder in folders: + if not isabs(default[folder]): + # Get the absolute path. + default[folder] = abspath(default[folder]) + + # Test the folder + if not exists(default[folder]): + print >> stderr, 'The folder %s does not exist.' % default[folder] + exit() + +# Test files +state_file = None +osm_file = None +mapping_file = None +post_import_file = None +for f in listdir(default['BASE_PBF']): + if f.endswith('.state.txt'): + state_file = join(default['BASE_PBF'], f) + + if f.endswith('.osm.pbf'): + osm_file = join(default['BASE_PBF'], f) + +if not osm_file: + print >> stderr, 'OSM file *.pbf is missing in %s' % default['BASE_PBF'] + exit() + +if not state_file: + print >> stderr, 'State file *.state.txt is missing in %s' % default['BASE_PBF'] + exit() + +for f in listdir(default['SETTINGS']): + if f.endswith('.json'): + mapping_file = join(default['SETTINGS'], f) + + if f.endswith('.sql'): + post_import_file = join(default['SETTINGS'], f) + +if not mapping_file: + print >> stderr, 'Mapping file *.json is missing in %s' % default['SETTINGS'] + exit() + +if not post_import_file: + print 'No *.sql detected in %s' % default['SETTINGS'] +else: + print '%s detected for post import.' % post_import_file + +# Check if there is a table starting with 'osm_' +sql = 'select count(*) ' \ + 'from information_schema.tables ' \ + 'where table_name like \'osm_%\';' +# noinspection PyUnboundLocalVariable +cursor.execute(sql) +osm_tables = cursor.fetchone()[0] +if osm_tables < 1: + # It means that the DB is empty. Let's import the file. + commands = ['imposm3', 'import', '-diff', '-deployproduction'] + commands += ['-overwritecache', '-cachedir', default['CACHE']] + commands += ['-srid', default['SRID']] + commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] + commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']] + commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] + commands += ['-diffdir', default['SETTINGS']] + commands += ['-mapping', mapping_file] + commands += ['-read', osm_file] + commands += ['-write', '-connection', postgis_uri] + + if not call(commands) == 0: + print >> stderr, 'An error occured in imposm with the original file.' + exit() + + if post_import_file: + for sql in open(post_import_file): + cursor.execute(sql) + +# Finally launch the listening process. +while True: + import_queue = sorted(listdir(default['IMPORT_QUEUE'])) + if len(import_queue) > 0: + for diff in import_queue: + print 'Importing diff %s' % diff + commands = ['imposm3', 'diff'] + commands += ['-cachedir', default['CACHE']] + commands += ['-dbschema-production', default['DBSCHEMA_PRODUCTION']] + commands += ['-dbschema-import', default['DBSCHEMA_IMPORT']] + commands += ['-dbschema-backup', default['DBSCHEMA_BACKUP']] + commands += ['-srid', default['SRID']] + commands += ['-diffdir', default['SETTINGS']] + commands += ['-mapping', mapping_file] + commands += ['-connection', postgis_uri] + commands += [join(default['IMPORT_QUEUE'], diff)] + + if call(commands) == 0: + move( + join(default['IMPORT_QUEUE'], diff), + join(default['IMPORT_DONE'], diff)) + else: + print >> stderr, 'An error occured in imposm with a diff.' + exit() + + if len(listdir(default['IMPORT_QUEUE'])) == 0: + print 'Sleeping for %s seconds.' % default['TIME'] + sleep(default['TIME']) diff --git a/docker-imposm3/requirements.txt b/docker-imposm3/requirements.txt new file mode 100644 index 0000000..ad7c893 --- /dev/null +++ b/docker-imposm3/requirements.txt @@ -0,0 +1,2 @@ +watchdog==0.8.3 +psycopg2 \ No newline at end of file diff --git a/osm-storage/Dockerfile b/docker-osm-storage/Dockerfile similarity index 100% rename from osm-storage/Dockerfile rename to docker-osm-storage/Dockerfile diff --git a/docker-osmupdate/.idea/.name b/docker-osmupdate/.idea/.name new file mode 100644 index 0000000..076c77d --- /dev/null +++ b/docker-osmupdate/.idea/.name @@ -0,0 +1 @@ +docker-osmupdate \ No newline at end of file diff --git a/docker-osmupdate/.idea/docker-osmupdate.iml b/docker-osmupdate/.idea/docker-osmupdate.iml new file mode 100644 index 0000000..1be76f1 --- /dev/null +++ b/docker-osmupdate/.idea/docker-osmupdate.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/encodings.xml b/docker-osmupdate/.idea/encodings.xml new file mode 100644 index 0000000..d821048 --- /dev/null +++ b/docker-osmupdate/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml b/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..86a287d --- /dev/null +++ b/docker-osmupdate/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml b/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..3b31283 --- /dev/null +++ b/docker-osmupdate/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/misc.xml b/docker-osmupdate/.idea/misc.xml new file mode 100644 index 0000000..7b4d854 --- /dev/null +++ b/docker-osmupdate/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/modules.xml b/docker-osmupdate/.idea/modules.xml new file mode 100644 index 0000000..b9e5349 --- /dev/null +++ b/docker-osmupdate/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/scopes/scope_settings.xml b/docker-osmupdate/.idea/scopes/scope_settings.xml new file mode 100644 index 0000000..922003b --- /dev/null +++ b/docker-osmupdate/.idea/scopes/scope_settings.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/vcs.xml b/docker-osmupdate/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/docker-osmupdate/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/docker-osmupdate/.idea/workspace.xml b/docker-osmupdate/.idea/workspace.xml new file mode 100644 index 0000000..c3bb32f --- /dev/null +++ b/docker-osmupdate/.idea/workspace.xml @@ -0,0 +1,682 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + 1437385466775 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker-osmupdate/71-apt-cacher-ng b/docker-osmupdate/71-apt-cacher-ng new file mode 100644 index 0000000..9502fe1 --- /dev/null +++ b/docker-osmupdate/71-apt-cacher-ng @@ -0,0 +1,2 @@ +#Change IP address to match that of your host +#Acquire::http { Proxy "http://192.168.10.200:3142"; }; diff --git a/docker-osmupdate/Dockerfile b/docker-osmupdate/Dockerfile new file mode 100644 index 0000000..c12aa62 --- /dev/null +++ b/docker-osmupdate/Dockerfile @@ -0,0 +1,29 @@ +#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ +FROM ubuntu:latest +MAINTAINER Etienne Trimaille + +RUN export DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND noninteractive +RUN dpkg-divert --local --rename --add /sbin/initctl + +# Use local cached debs from host (saves your bandwidth!) +# Change ip below to that of your apt-cacher-ng host +# Or comment this line out if you do not with to use caching +ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng + +# RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list +RUN apt-get -y update +RUN apt-get -y install ca-certificates rpl pwgen + +#-------------Application Specific Stuff ---------------------------------------------------- + +# Install osmupdate +RUN apt-get -y install osmctools wget gzip gcc libc-dev zlib1g-dev +WORKDIR /home +ADD osmupdate.c /home/osmupdate.c +RUN gcc -x c - -o osmupdate osmupdate.c + +# Add the python script which will call osmupdate +ADD download.py /home/download.py + +CMD ["python", "-u", "/home/download.py"] \ No newline at end of file diff --git a/docker-osmupdate/download.py b/docker-osmupdate/download.py new file mode 100644 index 0000000..c700ed7 --- /dev/null +++ b/docker-osmupdate/download.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +from os.path import exists, join, isabs, abspath +from os import listdir, environ +from sys import exit +from subprocess import call +from datetime import datetime +from time import sleep +from sys import stderr + +# Default values which can be overwritten. +default = { + 'MAX_DAYS': '100', + 'DIFF': 'sporadic', + 'MAX_MERGE': '7', + 'COMPRESSION_LEVEL': '1', + 'BASE_URL': 'http://planet.openstreetmap.org/replication/', + 'IMPORT_QUEUE': 'import_queue', + 'IMPORT_DONE': 'import_done', + 'BASE_PBF': 'base_pbf', + 'TIME': 120, +} + +for key in default.keys(): + if key in environ: + default[key] = environ[key] + +# Folders +folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'BASE_PBF'] +for folder in folders: + if not isabs(default[folder]): + # Get the absolute path. + default[folder] = abspath(default[folder]) + + # Test the folder + if not exists(default[folder]): + print >> stderr, 'The folder %s does not exist.' % default[folder] + exit() + +# Test files +state_file = None +osm_file = None +poly_file = None +for f in listdir(default['BASE_PBF']): + if f.endswith('.state.txt'): + state_file = join(default['BASE_PBF'], f) + + if f.endswith('.osm.pbf'): + osm_file = join(default['BASE_PBF'], f) + + if f.endswith('.poly'): + poly_file = join(default['BASE_PBF'], f) + +if not state_file: + print >> stderr, 'State file *.state.txt is missing in %s' % default['BASE_PBF'] + exit() + +if not osm_file: + print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['BASE_PBF'] + exit() + +if not poly_file: + print 'No *.poly detected in %s' % default['BASE_PBF'] +else: + print '%s detected for clipping.' % poly_file + +while True: + # Check if diff to be imported is empty. If not, take the latest diff. + diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE'])) + if len(diff_to_be_imported): + print "Timestamp from the lastest not imported diff." + timestamp = diff_to_be_imported[-1].split('.')[0] + else: + # Check if imported diff is empty. If not, take the latest diff. + imported_diff = sorted(listdir(default['IMPORT_DONE'])) + if len(imported_diff): + print "Timestamp from the lastest imported diff." + timestamp = imported_diff[-1].split('.')[0] + + else: + # Take the timestamp from original file. + print "Timestamp from the original state file." + state_file_settings = {} + with open(state_file) as a_file: + for line in a_file: + if '=' in line: + name, value = line.partition("=")[::2] + state_file_settings[name] = value + + timestamp = state_file_settings['timestamp'].strip() + + # Removing some \ in the timestamp. + timestamp = timestamp.replace('\\', '') + + # Save time + current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + print 'Old time : %s' % timestamp + print 'Current time : %s' % current_time + + # Destination + file_name = '%s.osc.gz' % current_time + file_path = join(default['IMPORT_QUEUE'], file_name) + + # Command + command = ['osmupdate', '-v'] + if poly_file: + command.append('-B=%s' % poly_file) + command += ['--max-days=' + default['MAX_DAYS']] + command += [default['DIFF']] + command += ['--max-merge=' + default['MAX_MERGE']] + command += ['--compression-level=' + default['COMPRESSION_LEVEL']] + command += ['--base-url=' + default['BASE_URL']] + command.append(timestamp) + command.append(file_path) + + if call(command) != 0: + print >> stderr, 'An error occured in osmupdate.' + + # Sleep + print 'Sleeping for %s seconds.' % default['TIME'] + sleep(default['TIME']) diff --git a/docker-osmupdate/osmupdate.c b/docker-osmupdate/osmupdate.c new file mode 100644 index 0000000..eef6de2 --- /dev/null +++ b/docker-osmupdate/osmupdate.c @@ -0,0 +1,1650 @@ +// osmupdate 2015-04-15 10:00 +#define VERSION "0.4.1" +// +// compile this file: +// gcc osmupdate.c -o osmupdate +// +// (c) 2011..2015 Markus Weber, Nuernberg +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU Affero General Public License +// version 3 as published by the Free Software Foundation. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// You should have received a copy of this license along +// with this program; if not, see http://www.gnu.org/licenses/. +// +// Other licenses are available on request; please ask the author. + +#define MAXLOGLEVEL 2 +const char* helptext= +"\nosmupdate " VERSION "\n" +"\n" +"This program cares about updating an .osm, .o5m or .pbf file. It\n" +"will download and apply OSM Change files (.osc) from the servers of\n" +"\"planet.openstreetmap.org\".\n" +"It also can assemble a new .osc or .o5c file which can be used to\n" +"update your OSM data file at a later time.\n" +"\n" +"Prequesites\n" +"\n" +"To run this program, please download and install two other programs\n" +"first: \"osmconvert\" and \"wget\".\n" +"\n" +"Usage\n" +"\n" +"Two command line arguments are mandatory: the name of the old and the\n" +"name of the new OSM data file. If the old data file does not have a\n" +"file timestamp, you may want to specify this timestamp manually on\n" +"the command line. If you do not, the program will try to determine\n" +"the timestamp by examining the whole old data file.\n" +"Instead of the second parameter, you alternatively may specify the\n" +"name of a change file (.osc or .o5c). In this case, you also may\n" +"replace the name of the old OSM data file by a timestamp.\n" +"Command line arguments which are not recognized by osmupdate will be\n" +"passed to osmconvert. Use this opportunity to supply a bounding box\n" +"or a bounding polygon if you are going to update a regional change\n" +"file. You also may exclude unneeded meta data from your file by\n" +"specifying this osmconvert option: --drop-author\n" +"\n" +"Usage Examples\n" +"\n" +" ./osmupdate old_file.o5m new_file.o5m\n" +" ./osmupdate old_file.pbf new_file.pbf\n" +" ./osmupdate old_file.osm new_file.osm\n" +" The old OSM data will be updated and written as new_file.o5m\n" +" or new_file.o5m. For safety reasons osmupdate will not delete\n" +" the old file. If you do not need it as backup file, please\n" +" delete it by yourself.\n" +"\n" +" ./osmupdate old_file.osm 2011-07-15T23:30:00Z new_file.osm\n" +" ./osmupdate old_file.osm NOW-86400 new_file.osm\n" +" If your old OSM data file does not contain a file timestamp,\n" +" or you do not want to rely on this timestamp, it can be\n" +" specified manually. Relative times are in seconds to NOW.\n" +"\n" +" ./osmupdate old_file.o5m change_file.o5c\n" +" ./osmupdate old_file.osm change_file.osc\n" +" ./osmupdate 2011-07-15T23:30:00Z change_file.o5c\n" +" ./osmupdate 2011-07-15T23:30:00Z change_file.osc.gz\n" +" ./osmupdate NOW-3600 change_file.osc.gz\n" +" Here, the old OSM data file is not updated directly. An OSM\n" +" changefile is written instead. This changefile can be used to\n" +" update the OSM data file afterwards.\n" +" You will have recognized the extension .gz in the last\n" +" example. In this case, the OSM Change file will be written\n" +" with gzip compression. To accomplish this, you need to have\n" +" the program gzip installed on your system.\n" +"\n" +" ./osmupdate london_old.o5m london_new.o5m -B=london.poly\n" +" The OSM data file london_old.o5m will be updated. Hence the\n" +" downloaded OSM changefiles contain not only London, but the\n" +" whole planet, a lot of unneeded data will be added to this\n" +" regional file. The -B= argument will clip these superfluous\n" +" data.\n" +"\n" +"The program osmupdate recognizes a few command line options:\n" +"\n" +"--max-days=UPDATE_RANGE\n" +" By default, the maximum time range for to assemble a\n" +" cumulated changefile is 250 days. You can change this by\n" +" giving a different maximum number of days, for example 300.\n" +" If you do, please ensure that there are daily change files\n" +" available for such a wide range of time.\n" +"\n" +"--minute\n" +"--hour\n" +"--day\n" +"--sporadic\n" +" By default, osmupdate uses a combination of minutely, hourly\n" +" and daily changefiles. If you want to limit these changefile\n" +" categories, use one or two of these options and choose that\n" +" category/ies you want to be used.\n" +" The option --sporadic allows processing changefile sources\n" +" which do not have the usual \"minute\", \"hour\" and \"day\"\n" +" subdirectories.\n" +"\n" +"--max-merge=COUNT\n" +" The subprogram osmconvert is able to merge more than two\n" +" changefiles in one run. This ability increases merging speed.\n" +" Unfortunately, every changefile consumes about 200 MB of main\n" +" memory while being processed. For this reason, the number of\n" +" parallely processable changefiles is limited.\n" +" Use this commandline argument to determine the maximum number\n" +" of parallely processed changefiles. The default value is 7.\n" +"\n" +"-t=TEMPPATH\n" +"--tempfiles=TEMPPATH\n" +" On order to cache changefiles, osmupdate needs a separate\n" +" directory. This parameter defines the name of this directory,\n" +" including the prefix of the tempfiles' names.\n" +" The default value is \"osmupdate_temp/temp\".\n" +"\n" +"--keep-tempfiles\n" +" Use this option if you want to keep local copies of every\n" +" downloaded file. This is strongly recommended if you are\n" +" going to assemble different changefiles which overlap in\n" +" time ranges. Your data traffic will be minimized.\n" +" Do not invoke this option if you are going to use different\n" +" change file sources (option --base-url). This would cause\n" +" severe data corruption.\n" +"\n" +"--compression-level=LEVEL\n" +" Define level for gzip compression. Values between 1 (low\n" +" compression, but fast) and 9 (high compression, but slow).\n" +"\n" +"--base-url=BASE_URL\n" +" To accelerate downloads or to get regional file updates you\n" +" may specify an alternative download location. Please enter\n" +" its URL, or simply the word \"mirror\" if you want to use\n" +" gwdg's planet server.\n" +"\n" +"--base-url-suffix=BASE_URL_SUFFIX\n" +" To use old planet URLs, you may need to add the suffix\n" +" \"-replicate\" because it was custom to have this word in the\n" +" URL, right after the period identifier \"day\" etc.\n" +"\n" +"-v\n" +"--verbose\n" +" With activated \'verbose\' mode, some statistical data and\n" +" diagnosis data will be displayed.\n" +" If -v resp. --verbose is the first parameter in the line,\n" +" osmupdate will display all input parameters.\n" +"\n" +"This program is for experimental use. Expect malfunctions and data\n" +"loss. Do not use the program in productive or commercial systems.\n" +"\n" +"There is NO WARRANTY, to the extent permitted by law.\n" +"Please send any bug reports to markus.weber@gmx.com\n\n"; + +#define _FILE_OFFSET_BITS 64 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef enum {false= 0,true= 1} bool; +typedef uint8_t byte; +typedef unsigned int uint; +#define isdig(x) isdigit((unsigned char)(x)) +static int loglevel= 0; // logging to stderr; + // 0: no logging; 1: small logging; 2: normal logging; + // 3: extended logging; +#define DP(f) fprintf(stderr,"Debug: " #f "\n"); +#define DPv(f,...) fprintf(stderr,"Debug: " #f "\n",__VA_ARGS__); +#define DPM(f,p,m) { byte* pp; int i,mm; static int msgn= 3; \ + if(--msgn>=0) { fprintf(stderr,"Debug memory: " #f); \ + pp= (byte*)(p); mm= (m); if(pp==NULL) fprintf(stderr,"\n (null)"); \ + else for(i= 0; i0 && *src!=0) + *d++= *src++; + *d= 0; + return dest; + } // end strmcpy() +#define strMcpy(d,s) strmcpy((d),(s),sizeof(d)) + +static inline char *stecpy(char** destp, char* destend, + const char* src) { + // same as stppcpy(), but you may define a pointer which the + // destination will not be allowed to cross, whether or not the + // string will be completed at that moment; + // in either case, the destination string will be terminated with 0; + char* dest; + + dest= *destp; + if(dest>=destend) +return dest; + destend--; + while(*src!=0 && dest=destend) +return dest; + destend-= 2; + while(*src!=0 && dest0 && *src!=0) + *d++= *src++; + *d= 0; + return d; + } // end stpmcpy() +#define stpMcpy(d,s) stpmcpy(d,s,sizeof(d)) + +static inline int strzcmp(const char* s1,const char* s2) { + // similar to strcmp(), this procedure compares two character strings; + // here, the number of characters which are to be compared is limited + // to the length of the second string; + // i.e., this procedure can be used to identify a short string s2 + // within a long string s1; + // s1[]: first string; + // s2[]: string to compare with the first string; + // return: + // 0: both strings are identical; the first string may be longer than + // the second; + // -1: the first string is alphabetical smaller than the second; + // 1: the first string is alphabetical greater than the second; + while(*s1==*s2 && *s1!=0) { s1++; s2++; } + if(*s2==0) + return 0; + return *(unsigned char*)s1 < *(unsigned char*)s2? -1: 1; + } // end strzcmp() + +static inline int strycmp(const char* s1,const char* s2) { + // similar to strcmp(), this procedure compares two character strings; + // here, both strings are end-aligned; + // not more characters will be compared than are existing in string s2; + // i.e., this procedure can be used to identify a file name extension; + const char* s1e; + int l; + + l= strchr(s2,0)-s2; + s1e= strchr(s1,0); + if(s1e-s1=10) + break; + i= i*10+b; + } + return i; + } // end strtouint32() + +static inline int64_t strtosint64(const char* s) { + // read a number and convert it to a signed 64-bit integer; + // return: number; + int sign; + int64_t i; + byte b; + + if(*s=='-') { s++; sign= -1; } else sign= 1; + i= 0; + for(;;) { + b= (byte)(*s++ -'0'); + if(b>=10) + break; + i= i*10+b; + } + return i*sign; + } // end strtosint64() + +static inline int64_t strtimetosint64(const char* s) { + // read a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", + // and convert it to a signed 64-bit integer; + // also allowed: relative time to NOW, e.g.: "NOW-86400", + // which means '24 hours ago'; + // return: time as a number (seconds since 1970); + // ==0: syntax error; + if(s[0]=='N') { // presumably a relative time to 'now' + if(s[1]!='O' || s[2]!='W' || (s[3]!='+' && s[3]!='-') || + !isdig(s[4])) // wrong syntax +return 0; + s+= 3; // jump over "NOW" + if(*s=='+') s++; // jump over '+', if any +return time(NULL)+strtosint64(s); + } // presumably a relative time to 'now' + if((s[0]!='1' && s[0]!='2') || + !isdig(s[1]) || !isdig(s[2]) || !isdig(s[3]) || + s[4]!='-' || !isdig(s[5]) || !isdig(s[6]) || + s[7]!='-' || !isdig(s[8]) || !isdig(s[9]) || + s[10]!='T' || !isdig(s[11]) || !isdig(s[12]) || + s[13]!=':' || !isdig(s[14]) || !isdig(s[15]) || + s[16]!=':' || !isdig(s[17]) || !isdig(s[18]) || + s[19]!='Z') // wrong syntax +return 0; + /* regular timestamp */ { + struct tm tm; + + tm.tm_isdst= 0; + tm.tm_year= + (s[0]-'0')*1000+(s[1]-'0')*100+(s[2]-'0')*10+(s[3]-'0')-1900; + tm.tm_mon= (s[5]-'0')*10+s[6]-'0'-1; + tm.tm_mday= (s[8]-'0')*10+s[9]-'0'; + tm.tm_hour= (s[11]-'0')*10+s[12]-'0'; + tm.tm_min= (s[14]-'0')*10+s[15]-'0'; + tm.tm_sec= (s[17]-'0')*10+s[18]-'0'; + #if __WIN32__ +return mktime(&tm)-timezone; + #else +return timegm(&tm); + #endif + } // regular timestamp + } // end strtimetosint64() + +static inline void int64tostrtime(uint64_t v,char* sp) { + // write a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", + // into a string; + // v: value of the timestamp; + // sp[21]: destination string; + time_t vtime; + struct tm tm; + int i; + + vtime= v; + #if __WIN32__ + memcpy(&tm,gmtime(&vtime),sizeof(tm)); + #else + gmtime_r(&vtime,&tm); + #endif + i= tm.tm_year+1900; + sp+= 3; *sp--= i%10+'0'; + i/=10; *sp--= i%10+'0'; + i/=10; *sp--= i%10+'0'; + i/=10; *sp= i%10+'0'; + sp+= 4; *sp++= '-'; + i= tm.tm_mon+1; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= '-'; + i= tm.tm_mday; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'T'; + i= tm.tm_hour; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; + i= tm.tm_min; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; + i= tm.tm_sec%60; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'Z'; *sp= 0; + } // end int64tostrtime() + +static inline bool file_exists(const char* file_name) { + // query if a file exists; + // file_name[]: name of the file in question; + // return: the file exists; + return access(file_name,R_OK)==0; + } // file_exists() + +static inline int64_t file_length(const char* file_name) { + // retrieve length of a file; + // file_name[]: file name; + // return: number of bytes of this file; + // if the file could not be accessed, the return value is -1; + struct stat s; + int r; + + r= stat(file_name,&s); + if(r==0) +return s.st_size; + #if !__WIN32__ + if(errno==EOVERFLOW) // size larger than 2^31 +return 0x7fffffff; + #endif + return -1; + } // end file_length() + + + +//------------------------------------------------------------ +// Module Global global variables for this program +//------------------------------------------------------------ + +// to distinguish global variable from local or module global +// variables, they are preceded by 'global_'; + +static const char global_osmconvert_program_here_in_dir[]= + "./osmconvert"; +static const char* global_osmconvert_program= + global_osmconvert_program_here_in_dir+2; + // path to osmconvert program +static char global_tempfile_name[450]= ""; + // prefix of names for temporary files +static bool global_keep_tempfiles= false; + // temporary files shall not be deleted at program end +static char global_osmconvert_arguments[2000]= ""; + // general command line arguments for osmconvert; +#define max_number_of_changefiles_in_cache 100 +static int global_max_merge= 7; + // maximum number of parallely processed changefiles +static const char* global_gzip_parameters= ""; + // parameters for gzip compression +static char global_base_url[400]= + "http://planet.openstreetmap.org/replication"; +static char global_base_url_suffix[100]=""; + // for old replication URL, to get "day-replication" instead of "day" + +#define PERR(f) \ + fprintf(stderr,"osmupdate Error: " f "\n"); + // print error message +#define PERRv(f,...) \ + fprintf(stderr,"osmupdate Error: " f "\n",__VA_ARGS__); + // print error message with value(s) +#define WARN(f) { static int msgn= 3; if(--msgn>=0) \ + fprintf(stderr,"osmupdate Warning: " f "\n"); } + // print a warning message, do it maximal 3 times +#define WARNv(f,...) { static int msgn= 3; if(--msgn>=0) \ + fprintf(stderr,"osmupdate Warning: " f "\n",__VA_ARGS__); } + // print a warning message with value(s), do it maximal 3 times +#define PINFO(f) \ + fprintf(stderr,"osmupdate: " f "\n"); // print info message +#define PINFOv(f,...) \ + fprintf(stderr,"osmupdate: " f "\n",__VA_ARGS__); +#define ONAME(i) \ + (i==0? "node": i==1? "way": i==2? "relation": "unknown object") + +//------------------------------------------------------------ +// end Module Global global variables for this program +//------------------------------------------------------------ + + + +static void shell_command(const char* command,char* result) { + // execute a shell command; + // command[]: shell command; + // result[1000]: result of the command; + FILE* fp; + char* result_p; + int maxlen; + int r; + + if(loglevel>=2) + PINFOv("Executing shell command:\n%s",command) + fp= popen(command,"r"); + if(fp==NULL) { + PERR("Could not execute shell command.") + result[0]= 0; +exit(1); + } + result_p= result; + maxlen= 1000-1; + while(maxlen>0) { + r= read(fileno(fp),result_p,maxlen); + if(r==0) // end of data + break; + if(r<0) +exit(errno); // (thanks to Ben Konrath) + result_p+= r; + maxlen-= r; + } + *result_p= 0; + if(pclose(fp)==-1) +exit(errno); // (thanks to Ben Konrath) + if(loglevel>=2) + PINFOv("Got shell command result:\n%s",result) + } // end shell_command() + +typedef enum {cft_UNKNOWN,cft_MINUTELY,cft_HOURLY,cft_DAILY,cft_SPORADIC} + changefile_type_t; +#define CFTNAME(i) \ + (i==cft_MINUTELY? "minutely": i==cft_HOURLY? "hourly": \ + i==cft_DAILY? "daily": i==cft_SPORADIC? "sporadic": "unknown") + +static int64_t get_file_timestamp(const char* file_name) { + // get the timestamp of a specific file; + // if the file timestamp is not available, this procedure tries + // to retrieve the timestamp from the file's statistics; + // return: timestamp of the file (seconds from Jan 01 1970); + // ==0: no file timestamp available; + char command[500],*command_p,result[1000]; + char* command_e= command+sizeof(command); + int64_t file_timestamp; + + command_p= command; + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," --out-timestamp \""); + steesccpy(&command_p,command_e,file_name); + stecpy(&command_p,command_e,"\" 2>&1"); + shell_command(command,result); + if(result[0]!='(' && (result[0]<'1' || result[0]>'2')) { + // command not found + PERR("Please install program osmconvert first.") +exit(1); + } // command not found + file_timestamp= strtimetosint64(result); + if(file_timestamp==0) { // the file has no file timestamp + // try to get the timestamp from the file's statistics + char* p; + + if(loglevel>0) { // verbose mode + PINFOv("file %s has no file timestamp.",file_name) + PINFO("Running statistics to get the timestamp.") + } + command_p= command; + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," --out-statistics \""); + steesccpy(&command_p,command_e,file_name); + stecpy(&command_p,command_e,"\" 2>&1"); + shell_command(command,result); + p= strstr(result,"timestamp max: "); + if(p!=NULL) { + file_timestamp= strtimetosint64(p+15); + PINFO("Aging the timestamp by 4 hours for safety reasons.") + file_timestamp-= 4*3600; + } + } // the file has no file timestamp + if(loglevel>0) { // verbose mode + char ts[30]; + + if(file_timestamp==0) + strcpy(ts,"(no timestamp)"); + else + int64tostrtime(file_timestamp,ts); + PINFOv("timestamp of %s: %s",file_name,ts) + } // verbose mode + return file_timestamp; + } // get_file_timestamp() + +static int64_t get_newest_changefile_timestamp( + changefile_type_t changefile_type,int32_t* file_sequence_number) { + // get sequence number and timestamp of the newest changefile + // of a specific changefile type; + // changefile_type: minutely, hourly, daily, sporadic changefile; + // return: timestamp of the file (seconds from Jan 01 1970); + // ==0: no file timestamp available; + // *file_sequence_number: sequence number of the newest changefile; + static bool firstrun= true; + char command[1000],*command_p; + char* command_e= command+sizeof(command); + char result[1000]; + int64_t changefile_timestamp; + + #if __WIN32__ + static char newest_timestamp_file_name[400]; + if(firstrun) { + // create the file name for the newest timestamp; + // this is only needed for Windows as Windows' wget + // cannot write the downloaded file to standard output; + // usually: "osmupdate_temp/temp.7" + strcpy(stpmcpy(newest_timestamp_file_name,global_tempfile_name, + sizeof(newest_timestamp_file_name)-5),".7"); + } + #endif + command_p= command; + stecpy(&command_p,command_e, + "wget -q "); + stecpy(&command_p,command_e,global_base_url); + switch(changefile_type) { // changefile type + case cft_MINUTELY: + stecpy(&command_p,command_e,"/minute"); + break; + case cft_HOURLY: + stecpy(&command_p,command_e,"/hour"); + break; + case cft_DAILY: + stecpy(&command_p,command_e,"/day"); + break; + case cft_SPORADIC: + break; + default: // invalid change file type +return 0; + } // changefile type + stecpy(&command_p,command_e,global_base_url_suffix); + stecpy(&command_p,command_e,"/state.txt"); + #if __WIN32__ + stecpy(&command_p,command_e," -O \""); + steesccpy(&command_p,command_e,newest_timestamp_file_name); + stecpy(&command_p,command_e,"\" 2>&1"); + #else + stecpy(&command_p,command_e," -O - 2>&1"); + #endif + shell_command(command,result); + if(firstrun) { // first run + firstrun= false; + if(result[0]!='#' && (result[0]<'1' || result[0]>'2') && ( + strstr(result,"not found")!=NULL || + strstr(result,"cannot find")!=NULL)) { // command not found + PERR("Please install program wget first.") +exit(1); + } // command not found + } // first run + #if __WIN32__ + result[0]= 0; + /* copy tempfile contents to result[] */ { + int fd,r; + + fd= open(newest_timestamp_file_name,O_RDONLY); + if(fd>0) { + r= read(fd,result,sizeof(result)-1); + if(r>=0) + result[r]= 0; + close(fd); + } + } + if(loglevel<2) + unlink(newest_timestamp_file_name); + #endif + if(result[0]=='#') { // full status information + // get sequence number + char* sequence_number_p; + sequence_number_p= strstr(result,"sequenceNumber="); + if(sequence_number_p!=NULL) // found sequence number + *file_sequence_number= strtouint32(sequence_number_p+15); + // get timestamp + char* timestamp_p; + timestamp_p= strstr(result,"timestamp="); + // search timestamp line + if(timestamp_p!=NULL && timestamp_p-result0) { // verbose mode + char ts[30]; + + if(changefile_timestamp==0) + strcpy(ts,"(no timestamp)"); + else + int64tostrtime(changefile_timestamp,ts); + PINFOv("newest %s timestamp: %s",CFTNAME(changefile_type),ts) + } // verbose mode +return changefile_timestamp; + } // get_newest_changefile_timestamp + +static int64_t get_changefile_timestamp( + changefile_type_t changefile_type,int32_t file_sequence_number) { + // download and inspect the timestamp of a specific changefile which + // is available in the Internet; + // a timestamp file will not be downloaded if it + // already exists locally as temporary file; + // changefile_type: minutely, hourly, daily, sporadic changefile; + // file_sequence_number: sequence number of the file; + // uses: + // global_tempfile_name + // return: timestamp of the changefile (seconds from Jan 01 1970); + // ==0: no file timestamp available; + char command[2000]; char* command_p; + char* command_e= command+sizeof(command); + char result[1000]; + char timestamp_cachefile_name[400]; + int fd,r; // file descriptor; number of bytes which have been read + char timestamp_contents[1000]; // contents of the timestamp + int64_t changefile_timestamp; + char* sp; + + // create the file name for the cached timestamp; example: + // "osmupdate_temp/temp.m000012345.txt" + sp= stpmcpy(timestamp_cachefile_name,global_tempfile_name, + sizeof(timestamp_cachefile_name[0])-20); + *sp++= '.'; + *sp++= CFTNAME(changefile_type)[0]; + // 'm', 'h', 'd', 's': minutely, hourly, daily, sporadic timestamps + sprintf(sp,"%09"PRIi32".txt",file_sequence_number); + // add sequence number and file name extension + + // download the timestamp into a cache file + if(file_length(timestamp_cachefile_name)<10) { + // timestamp has not been downloaded yet + command_p= command; + stecpy(&command_p,command_e,"wget -nv -c "); + stecpy(&command_p,command_e,global_base_url); + if(changefile_type==cft_MINUTELY) + stecpy(&command_p,command_e,"/minute"); + else if(changefile_type==cft_HOURLY) + stecpy(&command_p,command_e,"/hour"); + else if(changefile_type==cft_DAILY) + stecpy(&command_p,command_e,"/day"); + else if(changefile_type==cft_SPORADIC) + ; + else // invalid change file type + return 0; + stecpy(&command_p,command_e,global_base_url_suffix); + stecpy(&command_p,command_e,"/"); + /* assemble Sequence path */ { + int l; + l= sprintf(command_p,"%03i/%03i/%03i", + file_sequence_number/1000000,file_sequence_number/1000%1000, + file_sequence_number%1000); + command_p+= l; + } + stecpy(&command_p,command_e,".state.txt -O \""); + steesccpy(&command_p,command_e,timestamp_cachefile_name); + stecpy(&command_p,command_e,"\" 2>&1"); + shell_command(command,result); + } // timestamp has not been downloaded yet + + // read the timestamp cache file + fd= open(timestamp_cachefile_name,O_RDONLY|O_BINARY); + if(fd<=0) // could not open the file + timestamp_contents[0]= 0; // hence we did not read anything + else { // could open the file + r= read(fd,timestamp_contents,sizeof(timestamp_contents)-1); + if(r<0) r= 0; + timestamp_contents[r]= 0; // set string terminator + close(fd); + } // could open the file + + // parse the timestamp information + if(timestamp_contents[0]=='#') { // full status information + // get timestamp + char* timestamp_p; + timestamp_p= strstr(timestamp_contents,"timestamp="); + // search timestamp line + if(timestamp_p!=NULL && + timestamp_p-timestamp_contents0) { // verbose mode + char ts[30]; + + if(changefile_timestamp==0) + strcpy(ts,"(no timestamp)"); + else + int64tostrtime(changefile_timestamp,ts); + PINFOv("%s changefile %i: %s", + CFTNAME(changefile_type),file_sequence_number,ts) + } // verbose mode + if(changefile_timestamp==0) { // no timestamp + if(file_sequence_number==0) // first file in repository + changefile_timestamp= 1; // set virtual very old timestamp + else { + PERRv("no timestamp for %s changefile %i.", + CFTNAME(changefile_type),file_sequence_number) +exit(1); + } + } // no timestamp +return changefile_timestamp; + } // get_changefile_timestamp + +static void process_changefile( + changefile_type_t changefile_type,int32_t file_sequence_number, + int64_t new_timestamp) { + // download and process a change file; + // change files will not be processed one by one, but cumulated + // until some files have been downloaded and then processed in a group; + // a file will not be downloaded if it already exists locally as + // temporary file; + // changefile_type: minutely, hourly, daily, sporadic changefile; + // file_sequence_number: sequence number of the file; + // ==0: process the remaining files which + // are waiting in the cache; cleanup; + // new_timestamp: timestamp of the new file which is to be created; + // ==0: the procedure will assume the newest of all + // timestamps which has been passed since the + // program has been started; + // uses: + // global_max_merge + // global_tempfile_name + static bool firstrun= true; + static int number_of_changefiles_in_cache= 0; + static int64_t newest_new_timestamp= 0; + static char master_cachefile_name[400]; + static char master_cachefile_name_temp[400]; + static char cachefile_name[max_number_of_changefiles_in_cache][400]; + char command[4000+200*max_number_of_changefiles_in_cache]; + char* command_e= command+sizeof(command); + char* command_p; + char result[1000]; + + if(firstrun) { + firstrun= false; + // create the file name for the cached master changefile; + // usually: "osmupdate_temp/temp.8" + strcpy(stpmcpy(master_cachefile_name,global_tempfile_name, + sizeof(master_cachefile_name)-5),".8"); + strcpy(stpmcpy(master_cachefile_name_temp,global_tempfile_name, + sizeof(master_cachefile_name_temp)-5),".9"); + unlink(master_cachefile_name); + unlink(master_cachefile_name_temp); + } + if(new_timestamp>newest_new_timestamp) + newest_new_timestamp= new_timestamp; + if(file_sequence_number!=0) { // changefile download requested + char* this_cachefile_name= + cachefile_name[number_of_changefiles_in_cache]; + int64_t old_file_length; + char* sp; + + // create the file name for the cached changefile; example: + // "osmupdate_temp/temp.m000012345.osc.gz" + sp= stpmcpy(this_cachefile_name,global_tempfile_name, + sizeof(cachefile_name[0])-20); + *sp++= '.'; + *sp++= CFTNAME(changefile_type)[0]; + // 'm', 'h', 'd', 's': minutely, hourly, daily, + // sporadic changefiles + sprintf(sp,"%09"PRIi32".osc.gz",file_sequence_number); + // add sequence number and file name extension + + // assemble the URL and download the changefile + old_file_length= file_length(this_cachefile_name); + if(loglevel>0 && old_file_length<10) + // verbose mode AND file not downloaded yet + PINFOv("%s changefile %i: downloading", + CFTNAME(changefile_type),file_sequence_number) + command_p= command; + stecpy(&command_p,command_e,"wget -nv -c "); + stecpy(&command_p,command_e,global_base_url); + switch(changefile_type) { // changefile type + case cft_MINUTELY: + stecpy(&command_p,command_e,"/minute"); + break; + case cft_HOURLY: + stecpy(&command_p,command_e,"/hour"); + break; + case cft_DAILY: + stecpy(&command_p,command_e,"/day"); + break; + case cft_SPORADIC: + break; + default: // invalid change file type + return; + } // changefile type + stecpy(&command_p,command_e,global_base_url_suffix); + stecpy(&command_p,command_e,"/"); + + /* process sequence number */ { + int l; + l= sprintf(command_p,"%03i/%03i/%03i.osc.gz", + file_sequence_number/1000000,file_sequence_number/1000%1000, + file_sequence_number%1000); + command_p+= l; + } // process sequence number + + stecpy(&command_p,command_e," -O \""); + steesccpy(&command_p,command_e,this_cachefile_name); + stecpy(&command_p,command_e,"\" 2>&1 && echo \"Wget Command Ok\""); + shell_command(command,result); + if(strstr(result,"Wget Command Ok")==NULL) { // download error + PERRv("Could not download %s changefile %i", + CFTNAME(changefile_type),file_sequence_number) + PINFOv("wget Error message:\n%s",result) +exit(1); + } + if(loglevel>0 && old_file_length>=10) { + // verbose mode AND file was already in cache + if(file_length(this_cachefile_name)!=old_file_length) + PINFOv("%s changefile %i: download completed", + CFTNAME(changefile_type),file_sequence_number) + else + PINFOv("%s changefile %i: already in cache", + CFTNAME(changefile_type),file_sequence_number) + } // verbose mode + number_of_changefiles_in_cache++; + } // changefile download requested + + if(number_of_changefiles_in_cache>=global_max_merge + || (file_sequence_number==0 && number_of_changefiles_in_cache>0)) { + // at least one change files must be merged + // merge all changefiles which are waiting in cache + if(loglevel>0) + PINFO("Merging changefiles.") + command_p= command; + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," --merge-versions "); + stecpy(&command_p,command_e,global_osmconvert_arguments); + while(number_of_changefiles_in_cache>0) { + // for all changefiles in cache + number_of_changefiles_in_cache--; + stecpy(&command_p,command_e," \""); + steesccpy(&command_p,command_e, + cachefile_name[number_of_changefiles_in_cache]); + stecpy(&command_p,command_e,"\""); + } // for all changefiles in cache + if(file_exists(master_cachefile_name)) { + stecpy(&command_p,command_e," \""); + steesccpy(&command_p,command_e,master_cachefile_name); + stecpy(&command_p,command_e,"\""); + } + if(newest_new_timestamp!=0) { + stecpy(&command_p,command_e," --timestamp="); + if(command_e-command_p>=30) + int64tostrtime(newest_new_timestamp,command_p); + command_p= strchr(command_p,0); + } + stecpy(&command_p,command_e," --out-o5c >\""); + steesccpy(&command_p,command_e,master_cachefile_name_temp); + stecpy(&command_p,command_e,"\""); + shell_command(command,result); + if(file_length(master_cachefile_name_temp)<10 || + strstr(result,"Error")!=NULL || + strstr(result,"error")!=NULL) { // merging failed + PERRv("Merging of changefiles failed:\n%s",command) + if(result[0]!=0) + PERRv("%s",result) +exit(1); + } // merging failed + unlink(master_cachefile_name); + rename(master_cachefile_name_temp,master_cachefile_name); + } // at lease one change files must be merged + } // process_changefile() + +#if !__WIN32__ +void sigcatcher(int sig) { + fprintf(stderr,"osmupdate: Output has been terminated.\n"); + exit(1); + } // end sigchatcher() +#endif + +int main(int argc,const char** argv) { + // main procedure; + // for the meaning of the calling line parameters please look at the + // contents of helptext[]; + + // variables for command line arguments + int main_return_value; + const char* a; // command line argument + char* osmconvert_arguments_p; + // pointer in global_osmconvert_arguments[] + char final_osmconvert_arguments[2000]; + // command line arguments for the final run of osmconvert; + char* final_osmconvert_arguments_p; + // pointer in final_osmconvert_arguments[] + const char* old_file; // name of the old OSM file + int64_t old_timestamp; // timestamp of the old OSM file + const char* new_file; // name of the new OSM file or OSM Change file + bool new_file_is_o5; // the new file is type .o5m or .o5c + bool new_file_is_pbf; // the new file is type .pbf + bool new_file_is_changefile; // the new file is a changefile + bool new_file_is_gz; // the new file is a gzip compressed + int64_t max_update_range; // maximum range for cumulating changefiles + // in order to update an OSM file; unit: seconds; + char tempfile_directory[400]; // directory for temporary files + bool process_minutely,process_hourly,process_daily,process_sporadic; + // if one of these variables is true, then only the chosen categories + // shall be processed; + bool no_minutely,no_hourly,no_daily; + // the category shall not be processed; + + // regular variables + int64_t minutely_timestamp,hourly_timestamp,daily_timestamp, + sporadic_timestamp; + // timestamps for changefiles which are available in the Internet; + // unit: seconds after Jan 01 1970; + int32_t minutely_sequence_number,hourly_sequence_number, + daily_sequence_number,sporadic_sequence_number; + int64_t timestamp; + int64_t next_timestamp; + + // care about clean-up procedures + #if !__WIN32__ + /* care about signal handler */ { + static struct sigaction siga; + + siga.sa_handler= sigcatcher; + sigemptyset(&siga.sa_mask); + siga.sa_flags= 0; + sigaction(SIGPIPE,&siga,NULL); + } + #endif + + // initializations + main_return_value= 0; // (default) + #if __WIN32__ + setmode(fileno(stdout),O_BINARY); + setmode(fileno(stdin),O_BINARY); + #endif + osmconvert_arguments_p= global_osmconvert_arguments; + final_osmconvert_arguments[0]= 0; + final_osmconvert_arguments_p= final_osmconvert_arguments; + old_file= NULL; + old_timestamp= 0; + new_file= NULL; + new_file_is_o5= false; + new_file_is_pbf= false; + new_file_is_changefile= false; + new_file_is_gz= false; + max_update_range= 250*86400; + process_minutely= process_hourly= process_daily= process_sporadic= + false; + no_minutely= no_hourly= no_daily= false; + if(file_exists(global_osmconvert_program)) + // must be Linux (no ".exe" at the end) AND + // osmconvert program seems to be in this directory + global_osmconvert_program= global_osmconvert_program_here_in_dir; + + // read command line parameters + if(argc<=1) { // no command line parameters given + fprintf(stderr,"osmupdate " VERSION "\n" + "Updates .osm and .o5m files, downloads .osc and o5c files.\n" + "To get detailed help, please enter: ./osmupdate -h\n"); +return 0; // end the program, because without having parameters + // we do not know what to do; + } + while(--argc>0) { // for every parameter in command line + argv++; // switch to next parameter; as the first one is just + // the program name, we must do this prior reading the + // first 'real' parameter; + a= argv[0]; + if(loglevel>0) // verbose mode + fprintf(stderr,"osmupdate Parameter: %.2000s\n",a); + if(strcmp(a,"-h")==0 || strcmp(a,"-help")==0 || + strcmp(a,"--help")==0) { // user wants help text + fprintf(stdout,"%s",helptext); // print help text + // (took "%s", to prevent oversensitive compiler reactions) +return 0; + } + if((strcmp(a,"-v")==0 || strcmp(a,"--verbose")==0 || + strzcmp(a,"-v=")==0 || strzcmp(a,"--verbose=")==0) && + loglevel==0) { // test mode - if not given already + char* sp; + + sp= strchr(a,'='); + if(sp!=NULL) loglevel= sp[1]-'0'; else loglevel= 1; + if(loglevel<1) loglevel= 1; + if(loglevel>MAXLOGLEVEL) loglevel= MAXLOGLEVEL; + if(a[1]=='-') { // must be "--verbose" and not "-v" + if(loglevel==1) + fprintf(stderr,"osmupdate: Verbose mode.\n"); + else + fprintf(stderr,"osmupdate: Verbose mode %i.\n",loglevel); + } + continue; // take next parameter + } + if(strzcmp(a,"--max-days=")==0) { // maximum update range + max_update_range= (int64_t)strtouint32(a+11)*86400; + continue; // take next parameter + } + if((strzcmp(a,"-t=")==0 || strzcmp(a,"--tempfiles=")==0) && + global_tempfile_name[0]==0) { + // user-defined prefix for names of temorary files + strmcpy(global_tempfile_name,strchr(a,'=')+1, + sizeof(global_tempfile_name)-50); + continue; // take next parameter + } + if(strzcmp(a,"--keep-tempfiles")==0) { + // temporary files shall not be deleted at program end + global_keep_tempfiles= true; + continue; // take next parameter + } + if(strzcmp(a,"--compression-level=")==0) { + // gzip compression level + static char gzip_par[3]= ""; + + if(a[20]<'1' || a[20]>'9' || a[21]!=0) { + PINFO("Range error. Changed to --compression-level=3") + gzip_par[0]= '-'; gzip_par[1]= '3'; gzip_par[2]= 0; + } + else { + gzip_par[0]= '-'; gzip_par[1]= a[20]; gzip_par[2]= 0; + global_gzip_parameters= gzip_par; + } + continue; // take next parameter + } + if(strzcmp(a,"--max-merge=")==0) { + // maximum number of parallely processed changefiles + global_max_merge= strtouint32(a+12); + if(global_max_merge<2) { + global_max_merge= 2; + PINFO("Range error. Increased to --max-merge=2") + } + if(global_max_merge>max_number_of_changefiles_in_cache) { + global_max_merge= max_number_of_changefiles_in_cache; + PINFOv("Range error. Decreased to --max-merge=%i", + max_number_of_changefiles_in_cache) + } + continue; // take next parameter + } + if(strzcmp(a,"--minute")==0) { // process minutely data + // accept "--minute" as well as old syntax "--minutely" + process_minutely= true; + continue; // take next parameter + } + if(strzcmp(a,"--hour")==0) { // process hourly data + // accept "--hour" as well as old syntax "--hourly" + process_hourly= true; + continue; // take next parameter + } + if(strcmp(a,"--day")==0 || strzcmp(a,"--daily")==0) { + // process daily data; + // accept "--day" as well as old syntax "--daily" + process_daily= true; + continue; // take next parameter + } + if(strcmp(a,"--sporadic")==0) { + // process sporadic data; + process_sporadic= true; + continue; // take next parameter + } + if((strzcmp(a,"--base-url=")==0 && a[11]!=0) || + (strzcmp(a,"--planet-url=")==0 && a[13]!=0)) { + // change base url + // the second option keyword is deprecated but still supported + const char* ap; + char* sp; + + ap= a+11; + if(a[2]=='p') ap+= 2; + if(strcmp(ap,"mirror")==0) + strcpy(global_base_url,"ftp://ftp5.gwdg.de/pub/misc/" + "openstreetmap/planet.openstreetmap.org/replication"); + else if(strstr(ap,"://")!=NULL) + strmcpy(global_base_url,ap,sizeof(global_base_url)-1); + else { + strcpy(global_base_url,"http://"); + strmcpy(global_base_url+7,ap,sizeof(global_base_url)-8); + } + sp= strchr(global_base_url,0); + if(sp>global_base_url && sp[-1]=='/') + sp[-1]= 0; + continue; // take next parameter + } + if(strzcmp(a,"--base-url-suffix=")==0 && a[20]!=0) { + // change base url suffix + strMcpy(global_base_url_suffix,a+18); + continue; // take next parameter + } + if(strzcmp(a,"--planet-url-suffix=")==0 && a[20]!=0) { + // change base url suffix (this option keyword is deprecated) + strMcpy(global_base_url_suffix,a+20); + continue; // take next parameter + } + if(a[0]=='-') { + // command line argument not recognized by osmupdate + // store it so we can pass it to osmconvert later + int len; + + len= strlen(a)+3; + if(osmconvert_arguments_p-global_osmconvert_arguments+len + >=sizeof(global_osmconvert_arguments) || + final_osmconvert_arguments_p-final_osmconvert_arguments+len + >=sizeof(final_osmconvert_arguments)) { + PERR("too many command line arguments for osmconvert.") +return 1; + } + if(strcmp(a,"--complete-ways")==0 || + strcmp(a,"--complex-ways")==0 || + strcmp(a,"--drop-brokenrefs")==0 || + strcmp(a,"--drop-broken-refs")==0) { + WARNv("option %.80s does not work with updates.",a) + continue; // take next parameter + } + if(strzcmp(a,"-b=")!=0 && strzcmp(a,"-B=")!=0) { + // not a bounding box and not a bounding polygon + *osmconvert_arguments_p++= ' '; + *osmconvert_arguments_p++= '\"'; + osmconvert_arguments_p= stpesccpy(osmconvert_arguments_p,a); + *osmconvert_arguments_p++= '\"'; + *osmconvert_arguments_p= 0; + } + *final_osmconvert_arguments_p++= ' '; + *final_osmconvert_arguments_p++= '\"'; + final_osmconvert_arguments_p= + stpesccpy(final_osmconvert_arguments_p,a); + *final_osmconvert_arguments_p++= '\"'; + *final_osmconvert_arguments_p= 0; + continue; // take next parameter + } + if(old_timestamp==0) { + old_timestamp= strtimetosint64(a); + if(old_timestamp!=0) // this is a valid timestamp + continue; // take next parameter + } + // here: parameter must be a file name + if(old_file==NULL && old_timestamp==0) { // name of the old file + old_file= a; + continue; // take next parameter + } + if(new_file==NULL) { // name of the new file + new_file= a; + new_file_is_o5= + strycmp(new_file,".o5m")==0 || strycmp(new_file,".o5c")==0 || + strycmp(new_file,".o5m.gz")==0 || + strycmp(new_file,".o5c.gz")==0; + new_file_is_pbf= + strycmp(new_file,".pbf")==0; + new_file_is_changefile= + strycmp(new_file,".osc")==0 || strycmp(new_file,".o5c")==0 || + strycmp(new_file,".osc.gz")==0 || + strycmp(new_file,".o5c.gz")==0; + new_file_is_gz= strycmp(new_file,".gz")==0; + continue; // take next parameter + } + } // end for every parameter in command line + + /* create tempfile directory for cached timestamps and changefiles */ { + char *sp; + + if(strlen(global_tempfile_name)<2) // not set yet + strcpy(global_tempfile_name,"osmupdate_temp"DIRSEPS"temp"); + // take default + sp= strchr(global_tempfile_name,0); + if(sp[-1]==DIRSEP) // it's a bare directory + strcpy(sp,"temp"); // add a file name prefix + strMcpy(tempfile_directory,global_tempfile_name); + sp= strrchr(tempfile_directory,DIRSEP); + // get last directory separator + if(sp!=NULL) *sp= 0; // if found any, cut the string here + #if __WIN32__ + mkdir(tempfile_directory); + #else + mkdir(tempfile_directory,0700); + #endif + } + + // get file timestamp of OSM input file + if(old_timestamp==0) { // no timestamp given by the user + if(old_file==NULL) { // no file name given for the old OSM file + PERR("Specify at least the old OSM file's name or its timestamp.") +return 1; + } + if(!file_exists(old_file)) { // old OSM file does not exist + PERRv("Old OSM file does not exist: %.80s",old_file); +return 1; + } + old_timestamp= get_file_timestamp(old_file); + if(old_timestamp==0) { + PERRv("Old OSM file does not contain a timestamp: %.80s",old_file); + PERR("Please specify the timestamp manually, e.g.: " + "2011-07-15T23:30:00Z"); +return 1; + } + } // end no timestamp given by the user + + // parameter consistency check + if(new_file==NULL) { + PERR("No output file was specified."); +return 1; + } + if(old_file!=NULL && strcmp(old_file,new_file)==0) { + PERR("Input file and output file are identical."); +return 1; + } + if(old_file==NULL && !new_file_is_changefile) { + PERR("If no old OSM file is specified, osmupdate can only " + "generate a changefile."); +return 1; + } + + // initialize sequence numbers and timestamps + minutely_sequence_number= hourly_sequence_number= + daily_sequence_number= sporadic_sequence_number= 0; + minutely_timestamp= hourly_timestamp= + daily_timestamp= sporadic_timestamp= 0; + + // care about user defined processing categories + if(process_minutely || process_hourly || + process_daily || process_sporadic) { + // user wants specific type(s) of chancefiles to be processed + if(!process_minutely) no_minutely= true; + if(!process_hourly) no_hourly= true; + if(!process_daily) no_daily= true; + } + else { + // try to get sporadic timestamp + sporadic_timestamp= get_newest_changefile_timestamp( + cft_SPORADIC,&sporadic_sequence_number); + if(sporadic_timestamp!=0) { + // there is a timestamp at the highest directory level, + // this must be a so-called sporadic timestamp + if(loglevel>0) { + PINFO("Found status information in base URL root.") + PINFO("Ignoring subdirectories \"minute\", \"hour\", \"day\".") + } + process_sporadic= true; // let's take it + no_minutely= no_hourly= no_daily= true; + } + } + + // get last timestamp for each, minutely, hourly, daily, + // and sporadic diff files + if(!no_minutely) { + minutely_timestamp= get_newest_changefile_timestamp( + cft_MINUTELY,&minutely_sequence_number); + if(minutely_timestamp==0) { + PERR("Could not get the newest minutely timestamp from the Internet.") +return 1; + } + } + if(!no_hourly) { + hourly_timestamp= get_newest_changefile_timestamp( + cft_HOURLY,&hourly_sequence_number); + if(hourly_timestamp==0) { + PERR("Could not get the newest hourly timestamp from the Internet.") + return 1; + } + } + if(!no_daily) { + daily_timestamp= get_newest_changefile_timestamp( + cft_DAILY,&daily_sequence_number); + if(daily_timestamp==0) { + PERR("Could not get the newest daily timestamp from the Internet.") + return 1; + } + } + if(process_sporadic && sporadic_timestamp==0) { + sporadic_timestamp= get_newest_changefile_timestamp( + cft_SPORADIC,&sporadic_sequence_number); + if(sporadic_timestamp==0) { + PERR("Could not get the newest sporadic timestamp " + "from the Internet.") + return 1; + } + } + + // check maximum update range + if(minutely_timestamp-old_timestamp>max_update_range) { + // update range too large + int days; + days= (int)((minutely_timestamp-old_timestamp+86399)/86400); + PERRv("Update range too large: %i days.",days) + PINFOv("To allow such a wide range, add: --max-days=%i",days) +return 1; + } // update range too large + + // clear last hourly timestamp if + // OSM old file's timestamp > latest hourly timestamp - 30 minutes + if(old_timestamp>hourly_timestamp-30*60 && !no_minutely) + hourly_timestamp= 0; // (let's take minutely updates) + + // clear last daily timestamp if + // OSM file timestamp > latest daily timestamp - 16 hours + if(old_timestamp>daily_timestamp-16*3600 && + !(no_hourly && no_minutely)) + daily_timestamp= 0; // (let's take hourly and minutely updates) + + // initialize start timestamp + timestamp= 0; + if(timestamphourly_timestamp && + next_timestamp>old_timestamp) { + timestamp= next_timestamp; + process_changefile(cft_MINUTELY,minutely_sequence_number,timestamp); + minutely_sequence_number--; + next_timestamp= get_changefile_timestamp( + cft_MINUTELY,minutely_sequence_number); + } + } + + // get and process hourly diff files from last hourly timestamp + // backward; stop just before last daily timestamp or OSM + // file timestamp has been reached; + if(hourly_timestamp!=0) { + next_timestamp= timestamp; + while(next_timestamp>daily_timestamp && + next_timestamp>old_timestamp) { + timestamp= next_timestamp; + process_changefile(cft_HOURLY,hourly_sequence_number,timestamp); + hourly_sequence_number--; + next_timestamp= get_changefile_timestamp( + cft_HOURLY,hourly_sequence_number); + } + } + + // get and process daily diff files from last daily timestamp + // backward; stop just before OSM file timestamp has been reached; + if(daily_timestamp!=0) { + next_timestamp= timestamp; + while(next_timestamp>old_timestamp) { + timestamp= next_timestamp; + process_changefile(cft_DAILY,daily_sequence_number,timestamp); + daily_sequence_number--; + next_timestamp= get_changefile_timestamp( + cft_DAILY,daily_sequence_number); + } + } + + // get and process sporadic diff files from last sporadic timestamp + // backward; stop just before OSM file timestamp has been reached; + if(sporadic_timestamp!=0) { + next_timestamp= timestamp; + while(next_timestamp>old_timestamp) { + timestamp= next_timestamp; + process_changefile( + cft_SPORADIC,sporadic_sequence_number,timestamp); + sporadic_sequence_number--; + next_timestamp= get_changefile_timestamp( + cft_SPORADIC,sporadic_sequence_number); + } + } + + // process remaining files which may still wait in the cache; + process_changefile(0,0,0); + + /* create requested output file */ { + char master_cachefile_name[400]; + char command[2000],*command_p; + char* command_e= command+sizeof(command); + char result[1000]; + + if(loglevel>0) + PINFO("Creating output file.") + strcpy(stpmcpy(master_cachefile_name,global_tempfile_name, + sizeof(master_cachefile_name)-5),".8"); + if(!file_exists(master_cachefile_name)) { + if(old_file==NULL) + PINFO("There is no changefile since this timestamp.") + else + PINFO("Your OSM file is already up-to-date.") +return 21; + } + command_p= command; + if(new_file_is_changefile) { // changefile + if(new_file_is_gz) { // compressed + if(new_file_is_o5) { // .o5c.gz + stecpy(&command_p,command_e,"gzip "); + stecpy(&command_p,command_e,global_gzip_parameters); + stecpy(&command_p,command_e," <\""); + steesccpy(&command_p,command_e,master_cachefile_name); + stecpy(&command_p,command_e,"\" >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + else { // .osc.gz + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," "); + stecpy(&command_p,command_e,global_osmconvert_arguments); + stecpy(&command_p,command_e," \""); + steesccpy(&command_p,command_e,master_cachefile_name); + stecpy(&command_p,command_e,"\" --out-osc |gzip "); + stecpy(&command_p,command_e,global_gzip_parameters); + stecpy(&command_p,command_e," >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + shell_command(command,result); + } // compressed + else { // uncompressed + if(new_file_is_o5) // .o5c + rename(master_cachefile_name,new_file); + else { // .osc + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," "); + stecpy(&command_p,command_e,global_osmconvert_arguments); + stecpy(&command_p,command_e," \""); + steesccpy(&command_p,command_e,master_cachefile_name); + stecpy(&command_p,command_e,"\" --out-osc >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + shell_command(command,result); + } + } // uncompressed + } // changefile + else { // OSM file + #if 0 + if(loglevel>=2) { + PINFOv("oc %s",global_osmconvert_program) + PINFOv("fa %s",final_osmconvert_arguments) + PINFOv("of %s",old_file) + PINFOv("mc %s",master_cachefile_name) + } + #endif + stecpy(&command_p,command_e,global_osmconvert_program); + stecpy(&command_p,command_e," "); + stecpy(&command_p,command_e,final_osmconvert_arguments); + stecpy(&command_p,command_e," \""); + steesccpy(&command_p,command_e,old_file); + stecpy(&command_p,command_e,"\" \""); + steesccpy(&command_p,command_e,master_cachefile_name); + if(new_file_is_gz) { // compressed + if(new_file_is_o5) { // .o5m.gz + stecpy(&command_p,command_e,"\" --out-o5m |gzip "); + stecpy(&command_p,command_e,global_gzip_parameters); + stecpy(&command_p,command_e," >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + else { // .osm.gz + stecpy(&command_p,command_e,"\" --out-osm |gzip "); + stecpy(&command_p,command_e,global_gzip_parameters); + stecpy(&command_p,command_e," >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + } // compressed + else { // uncompressed + if(new_file_is_pbf) { // .pbf + stecpy(&command_p,command_e,"\" --out-pbf >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + else if(new_file_is_o5) { // .o5m + stecpy(&command_p,command_e,"\" --out-o5m >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + else { // .osm + stecpy(&command_p,command_e,"\" --out-osm >\""); + steesccpy(&command_p,command_e,new_file); + stecpy(&command_p,command_e,"\""); + } + } // uncompressed + shell_command(command,result); + } // OSM file + if(loglevel<2) + unlink(master_cachefile_name); + } // create requested output file + + // delete tempfiles + if(global_keep_tempfiles) { // tempfiles shall be kept + if(loglevel>0) + PINFO("Keeping temporary files.") + } // tempfiles shall be kept + else { // tempfiles shall be deleted + char command[500],*command_p,result[1000]; + char* command_e= command+sizeof(command); + + if(loglevel>0) + PINFO("Deleting temporary files.") + command_p= command; + stecpy(&command_p,command_e,DELFILE" \""); + steesccpy(&command_p,command_e,global_tempfile_name); + stecpy(&command_p,command_e,"\".*"); + shell_command(command,result); + rmdir(tempfile_directory); + } // tempfiles shall be deleted + + if(main_return_value==0 && loglevel>0) + PINFO("Completed successfully.") + + return main_return_value; + } // end main() + diff --git a/docker-osmupdate/readme.md b/docker-osmupdate/readme.md new file mode 100644 index 0000000..83bb263 --- /dev/null +++ b/docker-osmupdate/readme.md @@ -0,0 +1,20 @@ +## Docker OSM Update + +You should have 3 folders : base_pbf, import_queue, import_done + +Put a state file in base-pbf like this one : +http://download.openstreetmap.fr/extracts/africa/south_africa.state.txt + +``docker build -t osmupdate .`` +``docker run -v $('pwd')import-queue/:/home/import-queue -v $('pwd')base-pbf/:/home/base-pbf -v $('pwd')import-done/:/home/import-done -d osmupdate`` + +With -e, you can add some settings : + - MAX_DAYS = 100, the maximum time range to assamble a cumulated changefile. + - DIFF = sporadic, osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic. + - MAX_MERGE = 7, argument to determine the maximum number of parallely processed changefiles. + - COMPRESSION_LEVEL = 1, define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow) + - BASE_URL = http://planet.openstreetmap.org/replication/, change the URL to use a custom URL to fetch regional file updates. + - IMPORT_QUEUE = import_queue + - IMPORT_DONE = import_done + - BASE_PBF = base_pbf + - TIME = 120, secondes between two executions of the script \ No newline at end of file diff --git a/settings/post-pbf-import.sql b/settings/post-pbf-import.sql new file mode 100644 index 0000000..e69de29