+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docker-osmupdate/71-apt-cacher-ng b/docker-osmupdate/71-apt-cacher-ng
new file mode 100644
index 0000000..9502fe1
--- /dev/null
+++ b/docker-osmupdate/71-apt-cacher-ng
@@ -0,0 +1,2 @@
+#Change IP address to match that of your host
+#Acquire::http { Proxy "http://192.168.10.200:3142"; };
diff --git a/docker-osmupdate/Dockerfile b/docker-osmupdate/Dockerfile
new file mode 100644
index 0000000..c12aa62
--- /dev/null
+++ b/docker-osmupdate/Dockerfile
@@ -0,0 +1,29 @@
+#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------
+FROM ubuntu:latest
+MAINTAINER Etienne Trimaille
+
+RUN export DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND noninteractive
+RUN dpkg-divert --local --rename --add /sbin/initctl
+
+# Use local cached debs from host (saves your bandwidth!)
+# Change ip below to that of your apt-cacher-ng host
+# Or comment this line out if you do not with to use caching
+ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng
+
+# RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
+RUN apt-get -y update
+RUN apt-get -y install ca-certificates rpl pwgen
+
+#-------------Application Specific Stuff ----------------------------------------------------
+
+# Install osmupdate
+RUN apt-get -y install osmctools wget gzip gcc libc-dev zlib1g-dev
+WORKDIR /home
+ADD osmupdate.c /home/osmupdate.c
+RUN gcc -x c - -o osmupdate osmupdate.c
+
+# Add the python script which will call osmupdate
+ADD download.py /home/download.py
+
+CMD ["python", "-u", "/home/download.py"]
\ No newline at end of file
diff --git a/docker-osmupdate/download.py b/docker-osmupdate/download.py
new file mode 100644
index 0000000..c700ed7
--- /dev/null
+++ b/docker-osmupdate/download.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+from os.path import exists, join, isabs, abspath
+from os import listdir, environ
+from sys import exit
+from subprocess import call
+from datetime import datetime
+from time import sleep
+from sys import stderr
+
+# Default values which can be overwritten.
+default = {
+ 'MAX_DAYS': '100',
+ 'DIFF': 'sporadic',
+ 'MAX_MERGE': '7',
+ 'COMPRESSION_LEVEL': '1',
+ 'BASE_URL': 'http://planet.openstreetmap.org/replication/',
+ 'IMPORT_QUEUE': 'import_queue',
+ 'IMPORT_DONE': 'import_done',
+ 'BASE_PBF': 'base_pbf',
+ 'TIME': 120,
+}
+
+for key in default.keys():
+ if key in environ:
+ default[key] = environ[key]
+
+# Folders
+folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'BASE_PBF']
+for folder in folders:
+ if not isabs(default[folder]):
+ # Get the absolute path.
+ default[folder] = abspath(default[folder])
+
+ # Test the folder
+ if not exists(default[folder]):
+ print >> stderr, 'The folder %s does not exist.' % default[folder]
+ exit()
+
+# Test files
+state_file = None
+osm_file = None
+poly_file = None
+for f in listdir(default['BASE_PBF']):
+ if f.endswith('.state.txt'):
+ state_file = join(default['BASE_PBF'], f)
+
+ if f.endswith('.osm.pbf'):
+ osm_file = join(default['BASE_PBF'], f)
+
+ if f.endswith('.poly'):
+ poly_file = join(default['BASE_PBF'], f)
+
+if not state_file:
+ print >> stderr, 'State file *.state.txt is missing in %s' % default['BASE_PBF']
+ exit()
+
+if not osm_file:
+ print >> stderr, 'OSM file *.osm.pbf is missing in %s' % default['BASE_PBF']
+ exit()
+
+if not poly_file:
+ print 'No *.poly detected in %s' % default['BASE_PBF']
+else:
+ print '%s detected for clipping.' % poly_file
+
+while True:
+ # Check if diff to be imported is empty. If not, take the latest diff.
+ diff_to_be_imported = sorted(listdir(default['IMPORT_QUEUE']))
+ if len(diff_to_be_imported):
+ print "Timestamp from the lastest not imported diff."
+ timestamp = diff_to_be_imported[-1].split('.')[0]
+ else:
+ # Check if imported diff is empty. If not, take the latest diff.
+ imported_diff = sorted(listdir(default['IMPORT_DONE']))
+ if len(imported_diff):
+ print "Timestamp from the lastest imported diff."
+ timestamp = imported_diff[-1].split('.')[0]
+
+ else:
+ # Take the timestamp from original file.
+ print "Timestamp from the original state file."
+ state_file_settings = {}
+ with open(state_file) as a_file:
+ for line in a_file:
+ if '=' in line:
+ name, value = line.partition("=")[::2]
+ state_file_settings[name] = value
+
+ timestamp = state_file_settings['timestamp'].strip()
+
+ # Removing some \ in the timestamp.
+ timestamp = timestamp.replace('\\', '')
+
+ # Save time
+ current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+ print 'Old time : %s' % timestamp
+ print 'Current time : %s' % current_time
+
+ # Destination
+ file_name = '%s.osc.gz' % current_time
+ file_path = join(default['IMPORT_QUEUE'], file_name)
+
+ # Command
+ command = ['osmupdate', '-v']
+ if poly_file:
+ command.append('-B=%s' % poly_file)
+ command += ['--max-days=' + default['MAX_DAYS']]
+ command += [default['DIFF']]
+ command += ['--max-merge=' + default['MAX_MERGE']]
+ command += ['--compression-level=' + default['COMPRESSION_LEVEL']]
+ command += ['--base-url=' + default['BASE_URL']]
+ command.append(timestamp)
+ command.append(file_path)
+
+ if call(command) != 0:
+ print >> stderr, 'An error occured in osmupdate.'
+
+ # Sleep
+ print 'Sleeping for %s seconds.' % default['TIME']
+ sleep(default['TIME'])
diff --git a/docker-osmupdate/osmupdate.c b/docker-osmupdate/osmupdate.c
new file mode 100644
index 0000000..eef6de2
--- /dev/null
+++ b/docker-osmupdate/osmupdate.c
@@ -0,0 +1,1650 @@
+// osmupdate 2015-04-15 10:00
+#define VERSION "0.4.1"
+//
+// compile this file:
+// gcc osmupdate.c -o osmupdate
+//
+// (c) 2011..2015 Markus Weber, Nuernberg
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Affero General Public License
+// version 3 as published by the Free Software Foundation.
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+// You should have received a copy of this license along
+// with this program; if not, see http://www.gnu.org/licenses/.
+//
+// Other licenses are available on request; please ask the author.
+
+#define MAXLOGLEVEL 2
+const char* helptext=
+"\nosmupdate " VERSION "\n"
+"\n"
+"This program cares about updating an .osm, .o5m or .pbf file. It\n"
+"will download and apply OSM Change files (.osc) from the servers of\n"
+"\"planet.openstreetmap.org\".\n"
+"It also can assemble a new .osc or .o5c file which can be used to\n"
+"update your OSM data file at a later time.\n"
+"\n"
+"Prequesites\n"
+"\n"
+"To run this program, please download and install two other programs\n"
+"first: \"osmconvert\" and \"wget\".\n"
+"\n"
+"Usage\n"
+"\n"
+"Two command line arguments are mandatory: the name of the old and the\n"
+"name of the new OSM data file. If the old data file does not have a\n"
+"file timestamp, you may want to specify this timestamp manually on\n"
+"the command line. If you do not, the program will try to determine\n"
+"the timestamp by examining the whole old data file.\n"
+"Instead of the second parameter, you alternatively may specify the\n"
+"name of a change file (.osc or .o5c). In this case, you also may\n"
+"replace the name of the old OSM data file by a timestamp.\n"
+"Command line arguments which are not recognized by osmupdate will be\n"
+"passed to osmconvert. Use this opportunity to supply a bounding box\n"
+"or a bounding polygon if you are going to update a regional change\n"
+"file. You also may exclude unneeded meta data from your file by\n"
+"specifying this osmconvert option: --drop-author\n"
+"\n"
+"Usage Examples\n"
+"\n"
+" ./osmupdate old_file.o5m new_file.o5m\n"
+" ./osmupdate old_file.pbf new_file.pbf\n"
+" ./osmupdate old_file.osm new_file.osm\n"
+" The old OSM data will be updated and written as new_file.o5m\n"
+" or new_file.o5m. For safety reasons osmupdate will not delete\n"
+" the old file. If you do not need it as backup file, please\n"
+" delete it by yourself.\n"
+"\n"
+" ./osmupdate old_file.osm 2011-07-15T23:30:00Z new_file.osm\n"
+" ./osmupdate old_file.osm NOW-86400 new_file.osm\n"
+" If your old OSM data file does not contain a file timestamp,\n"
+" or you do not want to rely on this timestamp, it can be\n"
+" specified manually. Relative times are in seconds to NOW.\n"
+"\n"
+" ./osmupdate old_file.o5m change_file.o5c\n"
+" ./osmupdate old_file.osm change_file.osc\n"
+" ./osmupdate 2011-07-15T23:30:00Z change_file.o5c\n"
+" ./osmupdate 2011-07-15T23:30:00Z change_file.osc.gz\n"
+" ./osmupdate NOW-3600 change_file.osc.gz\n"
+" Here, the old OSM data file is not updated directly. An OSM\n"
+" changefile is written instead. This changefile can be used to\n"
+" update the OSM data file afterwards.\n"
+" You will have recognized the extension .gz in the last\n"
+" example. In this case, the OSM Change file will be written\n"
+" with gzip compression. To accomplish this, you need to have\n"
+" the program gzip installed on your system.\n"
+"\n"
+" ./osmupdate london_old.o5m london_new.o5m -B=london.poly\n"
+" The OSM data file london_old.o5m will be updated. Hence the\n"
+" downloaded OSM changefiles contain not only London, but the\n"
+" whole planet, a lot of unneeded data will be added to this\n"
+" regional file. The -B= argument will clip these superfluous\n"
+" data.\n"
+"\n"
+"The program osmupdate recognizes a few command line options:\n"
+"\n"
+"--max-days=UPDATE_RANGE\n"
+" By default, the maximum time range for to assemble a\n"
+" cumulated changefile is 250 days. You can change this by\n"
+" giving a different maximum number of days, for example 300.\n"
+" If you do, please ensure that there are daily change files\n"
+" available for such a wide range of time.\n"
+"\n"
+"--minute\n"
+"--hour\n"
+"--day\n"
+"--sporadic\n"
+" By default, osmupdate uses a combination of minutely, hourly\n"
+" and daily changefiles. If you want to limit these changefile\n"
+" categories, use one or two of these options and choose that\n"
+" category/ies you want to be used.\n"
+" The option --sporadic allows processing changefile sources\n"
+" which do not have the usual \"minute\", \"hour\" and \"day\"\n"
+" subdirectories.\n"
+"\n"
+"--max-merge=COUNT\n"
+" The subprogram osmconvert is able to merge more than two\n"
+" changefiles in one run. This ability increases merging speed.\n"
+" Unfortunately, every changefile consumes about 200 MB of main\n"
+" memory while being processed. For this reason, the number of\n"
+" parallely processable changefiles is limited.\n"
+" Use this commandline argument to determine the maximum number\n"
+" of parallely processed changefiles. The default value is 7.\n"
+"\n"
+"-t=TEMPPATH\n"
+"--tempfiles=TEMPPATH\n"
+" On order to cache changefiles, osmupdate needs a separate\n"
+" directory. This parameter defines the name of this directory,\n"
+" including the prefix of the tempfiles' names.\n"
+" The default value is \"osmupdate_temp/temp\".\n"
+"\n"
+"--keep-tempfiles\n"
+" Use this option if you want to keep local copies of every\n"
+" downloaded file. This is strongly recommended if you are\n"
+" going to assemble different changefiles which overlap in\n"
+" time ranges. Your data traffic will be minimized.\n"
+" Do not invoke this option if you are going to use different\n"
+" change file sources (option --base-url). This would cause\n"
+" severe data corruption.\n"
+"\n"
+"--compression-level=LEVEL\n"
+" Define level for gzip compression. Values between 1 (low\n"
+" compression, but fast) and 9 (high compression, but slow).\n"
+"\n"
+"--base-url=BASE_URL\n"
+" To accelerate downloads or to get regional file updates you\n"
+" may specify an alternative download location. Please enter\n"
+" its URL, or simply the word \"mirror\" if you want to use\n"
+" gwdg's planet server.\n"
+"\n"
+"--base-url-suffix=BASE_URL_SUFFIX\n"
+" To use old planet URLs, you may need to add the suffix\n"
+" \"-replicate\" because it was custom to have this word in the\n"
+" URL, right after the period identifier \"day\" etc.\n"
+"\n"
+"-v\n"
+"--verbose\n"
+" With activated \'verbose\' mode, some statistical data and\n"
+" diagnosis data will be displayed.\n"
+" If -v resp. --verbose is the first parameter in the line,\n"
+" osmupdate will display all input parameters.\n"
+"\n"
+"This program is for experimental use. Expect malfunctions and data\n"
+"loss. Do not use the program in productive or commercial systems.\n"
+"\n"
+"There is NO WARRANTY, to the extent permitted by law.\n"
+"Please send any bug reports to markus.weber@gmx.com\n\n";
+
+#define _FILE_OFFSET_BITS 64
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+typedef enum {false= 0,true= 1} bool;
+typedef uint8_t byte;
+typedef unsigned int uint;
+#define isdig(x) isdigit((unsigned char)(x))
+static int loglevel= 0; // logging to stderr;
+ // 0: no logging; 1: small logging; 2: normal logging;
+ // 3: extended logging;
+#define DP(f) fprintf(stderr,"Debug: " #f "\n");
+#define DPv(f,...) fprintf(stderr,"Debug: " #f "\n",__VA_ARGS__);
+#define DPM(f,p,m) { byte* pp; int i,mm; static int msgn= 3; \
+ if(--msgn>=0) { fprintf(stderr,"Debug memory: " #f); \
+ pp= (byte*)(p); mm= (m); if(pp==NULL) fprintf(stderr,"\n (null)"); \
+ else for(i= 0; i0 && *src!=0)
+ *d++= *src++;
+ *d= 0;
+ return dest;
+ } // end strmcpy()
+#define strMcpy(d,s) strmcpy((d),(s),sizeof(d))
+
+static inline char *stecpy(char** destp, char* destend,
+ const char* src) {
+ // same as stppcpy(), but you may define a pointer which the
+ // destination will not be allowed to cross, whether or not the
+ // string will be completed at that moment;
+ // in either case, the destination string will be terminated with 0;
+ char* dest;
+
+ dest= *destp;
+ if(dest>=destend)
+return dest;
+ destend--;
+ while(*src!=0 && dest=destend)
+return dest;
+ destend-= 2;
+ while(*src!=0 && dest0 && *src!=0)
+ *d++= *src++;
+ *d= 0;
+ return d;
+ } // end stpmcpy()
+#define stpMcpy(d,s) stpmcpy(d,s,sizeof(d))
+
+static inline int strzcmp(const char* s1,const char* s2) {
+ // similar to strcmp(), this procedure compares two character strings;
+ // here, the number of characters which are to be compared is limited
+ // to the length of the second string;
+ // i.e., this procedure can be used to identify a short string s2
+ // within a long string s1;
+ // s1[]: first string;
+ // s2[]: string to compare with the first string;
+ // return:
+ // 0: both strings are identical; the first string may be longer than
+ // the second;
+ // -1: the first string is alphabetical smaller than the second;
+ // 1: the first string is alphabetical greater than the second;
+ while(*s1==*s2 && *s1!=0) { s1++; s2++; }
+ if(*s2==0)
+ return 0;
+ return *(unsigned char*)s1 < *(unsigned char*)s2? -1: 1;
+ } // end strzcmp()
+
+static inline int strycmp(const char* s1,const char* s2) {
+ // similar to strcmp(), this procedure compares two character strings;
+ // here, both strings are end-aligned;
+ // not more characters will be compared than are existing in string s2;
+ // i.e., this procedure can be used to identify a file name extension;
+ const char* s1e;
+ int l;
+
+ l= strchr(s2,0)-s2;
+ s1e= strchr(s1,0);
+ if(s1e-s1=10)
+ break;
+ i= i*10+b;
+ }
+ return i;
+ } // end strtouint32()
+
+static inline int64_t strtosint64(const char* s) {
+ // read a number and convert it to a signed 64-bit integer;
+ // return: number;
+ int sign;
+ int64_t i;
+ byte b;
+
+ if(*s=='-') { s++; sign= -1; } else sign= 1;
+ i= 0;
+ for(;;) {
+ b= (byte)(*s++ -'0');
+ if(b>=10)
+ break;
+ i= i*10+b;
+ }
+ return i*sign;
+ } // end strtosint64()
+
+static inline int64_t strtimetosint64(const char* s) {
+ // read a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z",
+ // and convert it to a signed 64-bit integer;
+ // also allowed: relative time to NOW, e.g.: "NOW-86400",
+ // which means '24 hours ago';
+ // return: time as a number (seconds since 1970);
+ // ==0: syntax error;
+ if(s[0]=='N') { // presumably a relative time to 'now'
+ if(s[1]!='O' || s[2]!='W' || (s[3]!='+' && s[3]!='-') ||
+ !isdig(s[4])) // wrong syntax
+return 0;
+ s+= 3; // jump over "NOW"
+ if(*s=='+') s++; // jump over '+', if any
+return time(NULL)+strtosint64(s);
+ } // presumably a relative time to 'now'
+ if((s[0]!='1' && s[0]!='2') ||
+ !isdig(s[1]) || !isdig(s[2]) || !isdig(s[3]) ||
+ s[4]!='-' || !isdig(s[5]) || !isdig(s[6]) ||
+ s[7]!='-' || !isdig(s[8]) || !isdig(s[9]) ||
+ s[10]!='T' || !isdig(s[11]) || !isdig(s[12]) ||
+ s[13]!=':' || !isdig(s[14]) || !isdig(s[15]) ||
+ s[16]!=':' || !isdig(s[17]) || !isdig(s[18]) ||
+ s[19]!='Z') // wrong syntax
+return 0;
+ /* regular timestamp */ {
+ struct tm tm;
+
+ tm.tm_isdst= 0;
+ tm.tm_year=
+ (s[0]-'0')*1000+(s[1]-'0')*100+(s[2]-'0')*10+(s[3]-'0')-1900;
+ tm.tm_mon= (s[5]-'0')*10+s[6]-'0'-1;
+ tm.tm_mday= (s[8]-'0')*10+s[9]-'0';
+ tm.tm_hour= (s[11]-'0')*10+s[12]-'0';
+ tm.tm_min= (s[14]-'0')*10+s[15]-'0';
+ tm.tm_sec= (s[17]-'0')*10+s[18]-'0';
+ #if __WIN32__
+return mktime(&tm)-timezone;
+ #else
+return timegm(&tm);
+ #endif
+ } // regular timestamp
+ } // end strtimetosint64()
+
+static inline void int64tostrtime(uint64_t v,char* sp) {
+ // write a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z",
+ // into a string;
+ // v: value of the timestamp;
+ // sp[21]: destination string;
+ time_t vtime;
+ struct tm tm;
+ int i;
+
+ vtime= v;
+ #if __WIN32__
+ memcpy(&tm,gmtime(&vtime),sizeof(tm));
+ #else
+ gmtime_r(&vtime,&tm);
+ #endif
+ i= tm.tm_year+1900;
+ sp+= 3; *sp--= i%10+'0';
+ i/=10; *sp--= i%10+'0';
+ i/=10; *sp--= i%10+'0';
+ i/=10; *sp= i%10+'0';
+ sp+= 4; *sp++= '-';
+ i= tm.tm_mon+1;
+ *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= '-';
+ i= tm.tm_mday;
+ *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'T';
+ i= tm.tm_hour;
+ *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':';
+ i= tm.tm_min;
+ *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':';
+ i= tm.tm_sec%60;
+ *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'Z'; *sp= 0;
+ } // end int64tostrtime()
+
+static inline bool file_exists(const char* file_name) {
+ // query if a file exists;
+ // file_name[]: name of the file in question;
+ // return: the file exists;
+ return access(file_name,R_OK)==0;
+ } // file_exists()
+
+static inline int64_t file_length(const char* file_name) {
+ // retrieve length of a file;
+ // file_name[]: file name;
+ // return: number of bytes of this file;
+ // if the file could not be accessed, the return value is -1;
+ struct stat s;
+ int r;
+
+ r= stat(file_name,&s);
+ if(r==0)
+return s.st_size;
+ #if !__WIN32__
+ if(errno==EOVERFLOW) // size larger than 2^31
+return 0x7fffffff;
+ #endif
+ return -1;
+ } // end file_length()
+
+
+
+//------------------------------------------------------------
+// Module Global global variables for this program
+//------------------------------------------------------------
+
+// to distinguish global variable from local or module global
+// variables, they are preceded by 'global_';
+
+static const char global_osmconvert_program_here_in_dir[]=
+ "./osmconvert";
+static const char* global_osmconvert_program=
+ global_osmconvert_program_here_in_dir+2;
+ // path to osmconvert program
+static char global_tempfile_name[450]= "";
+ // prefix of names for temporary files
+static bool global_keep_tempfiles= false;
+ // temporary files shall not be deleted at program end
+static char global_osmconvert_arguments[2000]= "";
+ // general command line arguments for osmconvert;
+#define max_number_of_changefiles_in_cache 100
+static int global_max_merge= 7;
+ // maximum number of parallely processed changefiles
+static const char* global_gzip_parameters= "";
+ // parameters for gzip compression
+static char global_base_url[400]=
+ "http://planet.openstreetmap.org/replication";
+static char global_base_url_suffix[100]="";
+ // for old replication URL, to get "day-replication" instead of "day"
+
+#define PERR(f) \
+ fprintf(stderr,"osmupdate Error: " f "\n");
+ // print error message
+#define PERRv(f,...) \
+ fprintf(stderr,"osmupdate Error: " f "\n",__VA_ARGS__);
+ // print error message with value(s)
+#define WARN(f) { static int msgn= 3; if(--msgn>=0) \
+ fprintf(stderr,"osmupdate Warning: " f "\n"); }
+ // print a warning message, do it maximal 3 times
+#define WARNv(f,...) { static int msgn= 3; if(--msgn>=0) \
+ fprintf(stderr,"osmupdate Warning: " f "\n",__VA_ARGS__); }
+ // print a warning message with value(s), do it maximal 3 times
+#define PINFO(f) \
+ fprintf(stderr,"osmupdate: " f "\n"); // print info message
+#define PINFOv(f,...) \
+ fprintf(stderr,"osmupdate: " f "\n",__VA_ARGS__);
+#define ONAME(i) \
+ (i==0? "node": i==1? "way": i==2? "relation": "unknown object")
+
+//------------------------------------------------------------
+// end Module Global global variables for this program
+//------------------------------------------------------------
+
+
+
+static void shell_command(const char* command,char* result) {
+ // execute a shell command;
+ // command[]: shell command;
+ // result[1000]: result of the command;
+ FILE* fp;
+ char* result_p;
+ int maxlen;
+ int r;
+
+ if(loglevel>=2)
+ PINFOv("Executing shell command:\n%s",command)
+ fp= popen(command,"r");
+ if(fp==NULL) {
+ PERR("Could not execute shell command.")
+ result[0]= 0;
+exit(1);
+ }
+ result_p= result;
+ maxlen= 1000-1;
+ while(maxlen>0) {
+ r= read(fileno(fp),result_p,maxlen);
+ if(r==0) // end of data
+ break;
+ if(r<0)
+exit(errno); // (thanks to Ben Konrath)
+ result_p+= r;
+ maxlen-= r;
+ }
+ *result_p= 0;
+ if(pclose(fp)==-1)
+exit(errno); // (thanks to Ben Konrath)
+ if(loglevel>=2)
+ PINFOv("Got shell command result:\n%s",result)
+ } // end shell_command()
+
+typedef enum {cft_UNKNOWN,cft_MINUTELY,cft_HOURLY,cft_DAILY,cft_SPORADIC}
+ changefile_type_t;
+#define CFTNAME(i) \
+ (i==cft_MINUTELY? "minutely": i==cft_HOURLY? "hourly": \
+ i==cft_DAILY? "daily": i==cft_SPORADIC? "sporadic": "unknown")
+
+static int64_t get_file_timestamp(const char* file_name) {
+ // get the timestamp of a specific file;
+ // if the file timestamp is not available, this procedure tries
+ // to retrieve the timestamp from the file's statistics;
+ // return: timestamp of the file (seconds from Jan 01 1970);
+ // ==0: no file timestamp available;
+ char command[500],*command_p,result[1000];
+ char* command_e= command+sizeof(command);
+ int64_t file_timestamp;
+
+ command_p= command;
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," --out-timestamp \"");
+ steesccpy(&command_p,command_e,file_name);
+ stecpy(&command_p,command_e,"\" 2>&1");
+ shell_command(command,result);
+ if(result[0]!='(' && (result[0]<'1' || result[0]>'2')) {
+ // command not found
+ PERR("Please install program osmconvert first.")
+exit(1);
+ } // command not found
+ file_timestamp= strtimetosint64(result);
+ if(file_timestamp==0) { // the file has no file timestamp
+ // try to get the timestamp from the file's statistics
+ char* p;
+
+ if(loglevel>0) { // verbose mode
+ PINFOv("file %s has no file timestamp.",file_name)
+ PINFO("Running statistics to get the timestamp.")
+ }
+ command_p= command;
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," --out-statistics \"");
+ steesccpy(&command_p,command_e,file_name);
+ stecpy(&command_p,command_e,"\" 2>&1");
+ shell_command(command,result);
+ p= strstr(result,"timestamp max: ");
+ if(p!=NULL) {
+ file_timestamp= strtimetosint64(p+15);
+ PINFO("Aging the timestamp by 4 hours for safety reasons.")
+ file_timestamp-= 4*3600;
+ }
+ } // the file has no file timestamp
+ if(loglevel>0) { // verbose mode
+ char ts[30];
+
+ if(file_timestamp==0)
+ strcpy(ts,"(no timestamp)");
+ else
+ int64tostrtime(file_timestamp,ts);
+ PINFOv("timestamp of %s: %s",file_name,ts)
+ } // verbose mode
+ return file_timestamp;
+ } // get_file_timestamp()
+
+static int64_t get_newest_changefile_timestamp(
+ changefile_type_t changefile_type,int32_t* file_sequence_number) {
+ // get sequence number and timestamp of the newest changefile
+ // of a specific changefile type;
+ // changefile_type: minutely, hourly, daily, sporadic changefile;
+ // return: timestamp of the file (seconds from Jan 01 1970);
+ // ==0: no file timestamp available;
+ // *file_sequence_number: sequence number of the newest changefile;
+ static bool firstrun= true;
+ char command[1000],*command_p;
+ char* command_e= command+sizeof(command);
+ char result[1000];
+ int64_t changefile_timestamp;
+
+ #if __WIN32__
+ static char newest_timestamp_file_name[400];
+ if(firstrun) {
+ // create the file name for the newest timestamp;
+ // this is only needed for Windows as Windows' wget
+ // cannot write the downloaded file to standard output;
+ // usually: "osmupdate_temp/temp.7"
+ strcpy(stpmcpy(newest_timestamp_file_name,global_tempfile_name,
+ sizeof(newest_timestamp_file_name)-5),".7");
+ }
+ #endif
+ command_p= command;
+ stecpy(&command_p,command_e,
+ "wget -q ");
+ stecpy(&command_p,command_e,global_base_url);
+ switch(changefile_type) { // changefile type
+ case cft_MINUTELY:
+ stecpy(&command_p,command_e,"/minute");
+ break;
+ case cft_HOURLY:
+ stecpy(&command_p,command_e,"/hour");
+ break;
+ case cft_DAILY:
+ stecpy(&command_p,command_e,"/day");
+ break;
+ case cft_SPORADIC:
+ break;
+ default: // invalid change file type
+return 0;
+ } // changefile type
+ stecpy(&command_p,command_e,global_base_url_suffix);
+ stecpy(&command_p,command_e,"/state.txt");
+ #if __WIN32__
+ stecpy(&command_p,command_e," -O \"");
+ steesccpy(&command_p,command_e,newest_timestamp_file_name);
+ stecpy(&command_p,command_e,"\" 2>&1");
+ #else
+ stecpy(&command_p,command_e," -O - 2>&1");
+ #endif
+ shell_command(command,result);
+ if(firstrun) { // first run
+ firstrun= false;
+ if(result[0]!='#' && (result[0]<'1' || result[0]>'2') && (
+ strstr(result,"not found")!=NULL ||
+ strstr(result,"cannot find")!=NULL)) { // command not found
+ PERR("Please install program wget first.")
+exit(1);
+ } // command not found
+ } // first run
+ #if __WIN32__
+ result[0]= 0;
+ /* copy tempfile contents to result[] */ {
+ int fd,r;
+
+ fd= open(newest_timestamp_file_name,O_RDONLY);
+ if(fd>0) {
+ r= read(fd,result,sizeof(result)-1);
+ if(r>=0)
+ result[r]= 0;
+ close(fd);
+ }
+ }
+ if(loglevel<2)
+ unlink(newest_timestamp_file_name);
+ #endif
+ if(result[0]=='#') { // full status information
+ // get sequence number
+ char* sequence_number_p;
+ sequence_number_p= strstr(result,"sequenceNumber=");
+ if(sequence_number_p!=NULL) // found sequence number
+ *file_sequence_number= strtouint32(sequence_number_p+15);
+ // get timestamp
+ char* timestamp_p;
+ timestamp_p= strstr(result,"timestamp=");
+ // search timestamp line
+ if(timestamp_p!=NULL && timestamp_p-result0) { // verbose mode
+ char ts[30];
+
+ if(changefile_timestamp==0)
+ strcpy(ts,"(no timestamp)");
+ else
+ int64tostrtime(changefile_timestamp,ts);
+ PINFOv("newest %s timestamp: %s",CFTNAME(changefile_type),ts)
+ } // verbose mode
+return changefile_timestamp;
+ } // get_newest_changefile_timestamp
+
+static int64_t get_changefile_timestamp(
+ changefile_type_t changefile_type,int32_t file_sequence_number) {
+ // download and inspect the timestamp of a specific changefile which
+ // is available in the Internet;
+ // a timestamp file will not be downloaded if it
+ // already exists locally as temporary file;
+ // changefile_type: minutely, hourly, daily, sporadic changefile;
+ // file_sequence_number: sequence number of the file;
+ // uses:
+ // global_tempfile_name
+ // return: timestamp of the changefile (seconds from Jan 01 1970);
+ // ==0: no file timestamp available;
+ char command[2000]; char* command_p;
+ char* command_e= command+sizeof(command);
+ char result[1000];
+ char timestamp_cachefile_name[400];
+ int fd,r; // file descriptor; number of bytes which have been read
+ char timestamp_contents[1000]; // contents of the timestamp
+ int64_t changefile_timestamp;
+ char* sp;
+
+ // create the file name for the cached timestamp; example:
+ // "osmupdate_temp/temp.m000012345.txt"
+ sp= stpmcpy(timestamp_cachefile_name,global_tempfile_name,
+ sizeof(timestamp_cachefile_name[0])-20);
+ *sp++= '.';
+ *sp++= CFTNAME(changefile_type)[0];
+ // 'm', 'h', 'd', 's': minutely, hourly, daily, sporadic timestamps
+ sprintf(sp,"%09"PRIi32".txt",file_sequence_number);
+ // add sequence number and file name extension
+
+ // download the timestamp into a cache file
+ if(file_length(timestamp_cachefile_name)<10) {
+ // timestamp has not been downloaded yet
+ command_p= command;
+ stecpy(&command_p,command_e,"wget -nv -c ");
+ stecpy(&command_p,command_e,global_base_url);
+ if(changefile_type==cft_MINUTELY)
+ stecpy(&command_p,command_e,"/minute");
+ else if(changefile_type==cft_HOURLY)
+ stecpy(&command_p,command_e,"/hour");
+ else if(changefile_type==cft_DAILY)
+ stecpy(&command_p,command_e,"/day");
+ else if(changefile_type==cft_SPORADIC)
+ ;
+ else // invalid change file type
+ return 0;
+ stecpy(&command_p,command_e,global_base_url_suffix);
+ stecpy(&command_p,command_e,"/");
+ /* assemble Sequence path */ {
+ int l;
+ l= sprintf(command_p,"%03i/%03i/%03i",
+ file_sequence_number/1000000,file_sequence_number/1000%1000,
+ file_sequence_number%1000);
+ command_p+= l;
+ }
+ stecpy(&command_p,command_e,".state.txt -O \"");
+ steesccpy(&command_p,command_e,timestamp_cachefile_name);
+ stecpy(&command_p,command_e,"\" 2>&1");
+ shell_command(command,result);
+ } // timestamp has not been downloaded yet
+
+ // read the timestamp cache file
+ fd= open(timestamp_cachefile_name,O_RDONLY|O_BINARY);
+ if(fd<=0) // could not open the file
+ timestamp_contents[0]= 0; // hence we did not read anything
+ else { // could open the file
+ r= read(fd,timestamp_contents,sizeof(timestamp_contents)-1);
+ if(r<0) r= 0;
+ timestamp_contents[r]= 0; // set string terminator
+ close(fd);
+ } // could open the file
+
+ // parse the timestamp information
+ if(timestamp_contents[0]=='#') { // full status information
+ // get timestamp
+ char* timestamp_p;
+ timestamp_p= strstr(timestamp_contents,"timestamp=");
+ // search timestamp line
+ if(timestamp_p!=NULL &&
+ timestamp_p-timestamp_contents0) { // verbose mode
+ char ts[30];
+
+ if(changefile_timestamp==0)
+ strcpy(ts,"(no timestamp)");
+ else
+ int64tostrtime(changefile_timestamp,ts);
+ PINFOv("%s changefile %i: %s",
+ CFTNAME(changefile_type),file_sequence_number,ts)
+ } // verbose mode
+ if(changefile_timestamp==0) { // no timestamp
+ if(file_sequence_number==0) // first file in repository
+ changefile_timestamp= 1; // set virtual very old timestamp
+ else {
+ PERRv("no timestamp for %s changefile %i.",
+ CFTNAME(changefile_type),file_sequence_number)
+exit(1);
+ }
+ } // no timestamp
+return changefile_timestamp;
+ } // get_changefile_timestamp
+
+static void process_changefile(
+ changefile_type_t changefile_type,int32_t file_sequence_number,
+ int64_t new_timestamp) {
+ // download and process a change file;
+ // change files will not be processed one by one, but cumulated
+ // until some files have been downloaded and then processed in a group;
+ // a file will not be downloaded if it already exists locally as
+ // temporary file;
+ // changefile_type: minutely, hourly, daily, sporadic changefile;
+ // file_sequence_number: sequence number of the file;
+ // ==0: process the remaining files which
+ // are waiting in the cache; cleanup;
+ // new_timestamp: timestamp of the new file which is to be created;
+ // ==0: the procedure will assume the newest of all
+ // timestamps which has been passed since the
+ // program has been started;
+ // uses:
+ // global_max_merge
+ // global_tempfile_name
+ static bool firstrun= true;
+ static int number_of_changefiles_in_cache= 0;
+ static int64_t newest_new_timestamp= 0;
+ static char master_cachefile_name[400];
+ static char master_cachefile_name_temp[400];
+ static char cachefile_name[max_number_of_changefiles_in_cache][400];
+ char command[4000+200*max_number_of_changefiles_in_cache];
+ char* command_e= command+sizeof(command);
+ char* command_p;
+ char result[1000];
+
+ if(firstrun) {
+ firstrun= false;
+ // create the file name for the cached master changefile;
+ // usually: "osmupdate_temp/temp.8"
+ strcpy(stpmcpy(master_cachefile_name,global_tempfile_name,
+ sizeof(master_cachefile_name)-5),".8");
+ strcpy(stpmcpy(master_cachefile_name_temp,global_tempfile_name,
+ sizeof(master_cachefile_name_temp)-5),".9");
+ unlink(master_cachefile_name);
+ unlink(master_cachefile_name_temp);
+ }
+ if(new_timestamp>newest_new_timestamp)
+ newest_new_timestamp= new_timestamp;
+ if(file_sequence_number!=0) { // changefile download requested
+ char* this_cachefile_name=
+ cachefile_name[number_of_changefiles_in_cache];
+ int64_t old_file_length;
+ char* sp;
+
+ // create the file name for the cached changefile; example:
+ // "osmupdate_temp/temp.m000012345.osc.gz"
+ sp= stpmcpy(this_cachefile_name,global_tempfile_name,
+ sizeof(cachefile_name[0])-20);
+ *sp++= '.';
+ *sp++= CFTNAME(changefile_type)[0];
+ // 'm', 'h', 'd', 's': minutely, hourly, daily,
+ // sporadic changefiles
+ sprintf(sp,"%09"PRIi32".osc.gz",file_sequence_number);
+ // add sequence number and file name extension
+
+ // assemble the URL and download the changefile
+ old_file_length= file_length(this_cachefile_name);
+ if(loglevel>0 && old_file_length<10)
+ // verbose mode AND file not downloaded yet
+ PINFOv("%s changefile %i: downloading",
+ CFTNAME(changefile_type),file_sequence_number)
+ command_p= command;
+ stecpy(&command_p,command_e,"wget -nv -c ");
+ stecpy(&command_p,command_e,global_base_url);
+ switch(changefile_type) { // changefile type
+ case cft_MINUTELY:
+ stecpy(&command_p,command_e,"/minute");
+ break;
+ case cft_HOURLY:
+ stecpy(&command_p,command_e,"/hour");
+ break;
+ case cft_DAILY:
+ stecpy(&command_p,command_e,"/day");
+ break;
+ case cft_SPORADIC:
+ break;
+ default: // invalid change file type
+ return;
+ } // changefile type
+ stecpy(&command_p,command_e,global_base_url_suffix);
+ stecpy(&command_p,command_e,"/");
+
+ /* process sequence number */ {
+ int l;
+ l= sprintf(command_p,"%03i/%03i/%03i.osc.gz",
+ file_sequence_number/1000000,file_sequence_number/1000%1000,
+ file_sequence_number%1000);
+ command_p+= l;
+ } // process sequence number
+
+ stecpy(&command_p,command_e," -O \"");
+ steesccpy(&command_p,command_e,this_cachefile_name);
+ stecpy(&command_p,command_e,"\" 2>&1 && echo \"Wget Command Ok\"");
+ shell_command(command,result);
+ if(strstr(result,"Wget Command Ok")==NULL) { // download error
+ PERRv("Could not download %s changefile %i",
+ CFTNAME(changefile_type),file_sequence_number)
+ PINFOv("wget Error message:\n%s",result)
+exit(1);
+ }
+ if(loglevel>0 && old_file_length>=10) {
+ // verbose mode AND file was already in cache
+ if(file_length(this_cachefile_name)!=old_file_length)
+ PINFOv("%s changefile %i: download completed",
+ CFTNAME(changefile_type),file_sequence_number)
+ else
+ PINFOv("%s changefile %i: already in cache",
+ CFTNAME(changefile_type),file_sequence_number)
+ } // verbose mode
+ number_of_changefiles_in_cache++;
+ } // changefile download requested
+
+ if(number_of_changefiles_in_cache>=global_max_merge
+ || (file_sequence_number==0 && number_of_changefiles_in_cache>0)) {
+ // at least one change files must be merged
+ // merge all changefiles which are waiting in cache
+ if(loglevel>0)
+ PINFO("Merging changefiles.")
+ command_p= command;
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," --merge-versions ");
+ stecpy(&command_p,command_e,global_osmconvert_arguments);
+ while(number_of_changefiles_in_cache>0) {
+ // for all changefiles in cache
+ number_of_changefiles_in_cache--;
+ stecpy(&command_p,command_e," \"");
+ steesccpy(&command_p,command_e,
+ cachefile_name[number_of_changefiles_in_cache]);
+ stecpy(&command_p,command_e,"\"");
+ } // for all changefiles in cache
+ if(file_exists(master_cachefile_name)) {
+ stecpy(&command_p,command_e," \"");
+ steesccpy(&command_p,command_e,master_cachefile_name);
+ stecpy(&command_p,command_e,"\"");
+ }
+ if(newest_new_timestamp!=0) {
+ stecpy(&command_p,command_e," --timestamp=");
+ if(command_e-command_p>=30)
+ int64tostrtime(newest_new_timestamp,command_p);
+ command_p= strchr(command_p,0);
+ }
+ stecpy(&command_p,command_e," --out-o5c >\"");
+ steesccpy(&command_p,command_e,master_cachefile_name_temp);
+ stecpy(&command_p,command_e,"\"");
+ shell_command(command,result);
+ if(file_length(master_cachefile_name_temp)<10 ||
+ strstr(result,"Error")!=NULL ||
+ strstr(result,"error")!=NULL) { // merging failed
+ PERRv("Merging of changefiles failed:\n%s",command)
+ if(result[0]!=0)
+ PERRv("%s",result)
+exit(1);
+ } // merging failed
+ unlink(master_cachefile_name);
+ rename(master_cachefile_name_temp,master_cachefile_name);
+ } // at lease one change files must be merged
+ } // process_changefile()
+
+#if !__WIN32__
+void sigcatcher(int sig) {
+ fprintf(stderr,"osmupdate: Output has been terminated.\n");
+ exit(1);
+ } // end sigchatcher()
+#endif
+
+int main(int argc,const char** argv) {
+ // main procedure;
+ // for the meaning of the calling line parameters please look at the
+ // contents of helptext[];
+
+ // variables for command line arguments
+ int main_return_value;
+ const char* a; // command line argument
+ char* osmconvert_arguments_p;
+ // pointer in global_osmconvert_arguments[]
+ char final_osmconvert_arguments[2000];
+ // command line arguments for the final run of osmconvert;
+ char* final_osmconvert_arguments_p;
+ // pointer in final_osmconvert_arguments[]
+ const char* old_file; // name of the old OSM file
+ int64_t old_timestamp; // timestamp of the old OSM file
+ const char* new_file; // name of the new OSM file or OSM Change file
+ bool new_file_is_o5; // the new file is type .o5m or .o5c
+ bool new_file_is_pbf; // the new file is type .pbf
+ bool new_file_is_changefile; // the new file is a changefile
+ bool new_file_is_gz; // the new file is a gzip compressed
+ int64_t max_update_range; // maximum range for cumulating changefiles
+ // in order to update an OSM file; unit: seconds;
+ char tempfile_directory[400]; // directory for temporary files
+ bool process_minutely,process_hourly,process_daily,process_sporadic;
+ // if one of these variables is true, then only the chosen categories
+ // shall be processed;
+ bool no_minutely,no_hourly,no_daily;
+ // the category shall not be processed;
+
+ // regular variables
+ int64_t minutely_timestamp,hourly_timestamp,daily_timestamp,
+ sporadic_timestamp;
+ // timestamps for changefiles which are available in the Internet;
+ // unit: seconds after Jan 01 1970;
+ int32_t minutely_sequence_number,hourly_sequence_number,
+ daily_sequence_number,sporadic_sequence_number;
+ int64_t timestamp;
+ int64_t next_timestamp;
+
+ // care about clean-up procedures
+ #if !__WIN32__
+ /* care about signal handler */ {
+ static struct sigaction siga;
+
+ siga.sa_handler= sigcatcher;
+ sigemptyset(&siga.sa_mask);
+ siga.sa_flags= 0;
+ sigaction(SIGPIPE,&siga,NULL);
+ }
+ #endif
+
+ // initializations
+ main_return_value= 0; // (default)
+ #if __WIN32__
+ setmode(fileno(stdout),O_BINARY);
+ setmode(fileno(stdin),O_BINARY);
+ #endif
+ osmconvert_arguments_p= global_osmconvert_arguments;
+ final_osmconvert_arguments[0]= 0;
+ final_osmconvert_arguments_p= final_osmconvert_arguments;
+ old_file= NULL;
+ old_timestamp= 0;
+ new_file= NULL;
+ new_file_is_o5= false;
+ new_file_is_pbf= false;
+ new_file_is_changefile= false;
+ new_file_is_gz= false;
+ max_update_range= 250*86400;
+ process_minutely= process_hourly= process_daily= process_sporadic=
+ false;
+ no_minutely= no_hourly= no_daily= false;
+ if(file_exists(global_osmconvert_program))
+ // must be Linux (no ".exe" at the end) AND
+ // osmconvert program seems to be in this directory
+ global_osmconvert_program= global_osmconvert_program_here_in_dir;
+
+ // read command line parameters
+ if(argc<=1) { // no command line parameters given
+ fprintf(stderr,"osmupdate " VERSION "\n"
+ "Updates .osm and .o5m files, downloads .osc and o5c files.\n"
+ "To get detailed help, please enter: ./osmupdate -h\n");
+return 0; // end the program, because without having parameters
+ // we do not know what to do;
+ }
+ while(--argc>0) { // for every parameter in command line
+ argv++; // switch to next parameter; as the first one is just
+ // the program name, we must do this prior reading the
+ // first 'real' parameter;
+ a= argv[0];
+ if(loglevel>0) // verbose mode
+ fprintf(stderr,"osmupdate Parameter: %.2000s\n",a);
+ if(strcmp(a,"-h")==0 || strcmp(a,"-help")==0 ||
+ strcmp(a,"--help")==0) { // user wants help text
+ fprintf(stdout,"%s",helptext); // print help text
+ // (took "%s", to prevent oversensitive compiler reactions)
+return 0;
+ }
+ if((strcmp(a,"-v")==0 || strcmp(a,"--verbose")==0 ||
+ strzcmp(a,"-v=")==0 || strzcmp(a,"--verbose=")==0) &&
+ loglevel==0) { // test mode - if not given already
+ char* sp;
+
+ sp= strchr(a,'=');
+ if(sp!=NULL) loglevel= sp[1]-'0'; else loglevel= 1;
+ if(loglevel<1) loglevel= 1;
+ if(loglevel>MAXLOGLEVEL) loglevel= MAXLOGLEVEL;
+ if(a[1]=='-') { // must be "--verbose" and not "-v"
+ if(loglevel==1)
+ fprintf(stderr,"osmupdate: Verbose mode.\n");
+ else
+ fprintf(stderr,"osmupdate: Verbose mode %i.\n",loglevel);
+ }
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--max-days=")==0) { // maximum update range
+ max_update_range= (int64_t)strtouint32(a+11)*86400;
+ continue; // take next parameter
+ }
+ if((strzcmp(a,"-t=")==0 || strzcmp(a,"--tempfiles=")==0) &&
+ global_tempfile_name[0]==0) {
+ // user-defined prefix for names of temorary files
+ strmcpy(global_tempfile_name,strchr(a,'=')+1,
+ sizeof(global_tempfile_name)-50);
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--keep-tempfiles")==0) {
+ // temporary files shall not be deleted at program end
+ global_keep_tempfiles= true;
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--compression-level=")==0) {
+ // gzip compression level
+ static char gzip_par[3]= "";
+
+ if(a[20]<'1' || a[20]>'9' || a[21]!=0) {
+ PINFO("Range error. Changed to --compression-level=3")
+ gzip_par[0]= '-'; gzip_par[1]= '3'; gzip_par[2]= 0;
+ }
+ else {
+ gzip_par[0]= '-'; gzip_par[1]= a[20]; gzip_par[2]= 0;
+ global_gzip_parameters= gzip_par;
+ }
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--max-merge=")==0) {
+ // maximum number of parallely processed changefiles
+ global_max_merge= strtouint32(a+12);
+ if(global_max_merge<2) {
+ global_max_merge= 2;
+ PINFO("Range error. Increased to --max-merge=2")
+ }
+ if(global_max_merge>max_number_of_changefiles_in_cache) {
+ global_max_merge= max_number_of_changefiles_in_cache;
+ PINFOv("Range error. Decreased to --max-merge=%i",
+ max_number_of_changefiles_in_cache)
+ }
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--minute")==0) { // process minutely data
+ // accept "--minute" as well as old syntax "--minutely"
+ process_minutely= true;
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--hour")==0) { // process hourly data
+ // accept "--hour" as well as old syntax "--hourly"
+ process_hourly= true;
+ continue; // take next parameter
+ }
+ if(strcmp(a,"--day")==0 || strzcmp(a,"--daily")==0) {
+ // process daily data;
+ // accept "--day" as well as old syntax "--daily"
+ process_daily= true;
+ continue; // take next parameter
+ }
+ if(strcmp(a,"--sporadic")==0) {
+ // process sporadic data;
+ process_sporadic= true;
+ continue; // take next parameter
+ }
+ if((strzcmp(a,"--base-url=")==0 && a[11]!=0) ||
+ (strzcmp(a,"--planet-url=")==0 && a[13]!=0)) {
+ // change base url
+ // the second option keyword is deprecated but still supported
+ const char* ap;
+ char* sp;
+
+ ap= a+11;
+ if(a[2]=='p') ap+= 2;
+ if(strcmp(ap,"mirror")==0)
+ strcpy(global_base_url,"ftp://ftp5.gwdg.de/pub/misc/"
+ "openstreetmap/planet.openstreetmap.org/replication");
+ else if(strstr(ap,"://")!=NULL)
+ strmcpy(global_base_url,ap,sizeof(global_base_url)-1);
+ else {
+ strcpy(global_base_url,"http://");
+ strmcpy(global_base_url+7,ap,sizeof(global_base_url)-8);
+ }
+ sp= strchr(global_base_url,0);
+ if(sp>global_base_url && sp[-1]=='/')
+ sp[-1]= 0;
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--base-url-suffix=")==0 && a[20]!=0) {
+ // change base url suffix
+ strMcpy(global_base_url_suffix,a+18);
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"--planet-url-suffix=")==0 && a[20]!=0) {
+ // change base url suffix (this option keyword is deprecated)
+ strMcpy(global_base_url_suffix,a+20);
+ continue; // take next parameter
+ }
+ if(a[0]=='-') {
+ // command line argument not recognized by osmupdate
+ // store it so we can pass it to osmconvert later
+ int len;
+
+ len= strlen(a)+3;
+ if(osmconvert_arguments_p-global_osmconvert_arguments+len
+ >=sizeof(global_osmconvert_arguments) ||
+ final_osmconvert_arguments_p-final_osmconvert_arguments+len
+ >=sizeof(final_osmconvert_arguments)) {
+ PERR("too many command line arguments for osmconvert.")
+return 1;
+ }
+ if(strcmp(a,"--complete-ways")==0 ||
+ strcmp(a,"--complex-ways")==0 ||
+ strcmp(a,"--drop-brokenrefs")==0 ||
+ strcmp(a,"--drop-broken-refs")==0) {
+ WARNv("option %.80s does not work with updates.",a)
+ continue; // take next parameter
+ }
+ if(strzcmp(a,"-b=")!=0 && strzcmp(a,"-B=")!=0) {
+ // not a bounding box and not a bounding polygon
+ *osmconvert_arguments_p++= ' ';
+ *osmconvert_arguments_p++= '\"';
+ osmconvert_arguments_p= stpesccpy(osmconvert_arguments_p,a);
+ *osmconvert_arguments_p++= '\"';
+ *osmconvert_arguments_p= 0;
+ }
+ *final_osmconvert_arguments_p++= ' ';
+ *final_osmconvert_arguments_p++= '\"';
+ final_osmconvert_arguments_p=
+ stpesccpy(final_osmconvert_arguments_p,a);
+ *final_osmconvert_arguments_p++= '\"';
+ *final_osmconvert_arguments_p= 0;
+ continue; // take next parameter
+ }
+ if(old_timestamp==0) {
+ old_timestamp= strtimetosint64(a);
+ if(old_timestamp!=0) // this is a valid timestamp
+ continue; // take next parameter
+ }
+ // here: parameter must be a file name
+ if(old_file==NULL && old_timestamp==0) { // name of the old file
+ old_file= a;
+ continue; // take next parameter
+ }
+ if(new_file==NULL) { // name of the new file
+ new_file= a;
+ new_file_is_o5=
+ strycmp(new_file,".o5m")==0 || strycmp(new_file,".o5c")==0 ||
+ strycmp(new_file,".o5m.gz")==0 ||
+ strycmp(new_file,".o5c.gz")==0;
+ new_file_is_pbf=
+ strycmp(new_file,".pbf")==0;
+ new_file_is_changefile=
+ strycmp(new_file,".osc")==0 || strycmp(new_file,".o5c")==0 ||
+ strycmp(new_file,".osc.gz")==0 ||
+ strycmp(new_file,".o5c.gz")==0;
+ new_file_is_gz= strycmp(new_file,".gz")==0;
+ continue; // take next parameter
+ }
+ } // end for every parameter in command line
+
+ /* create tempfile directory for cached timestamps and changefiles */ {
+ char *sp;
+
+ if(strlen(global_tempfile_name)<2) // not set yet
+ strcpy(global_tempfile_name,"osmupdate_temp"DIRSEPS"temp");
+ // take default
+ sp= strchr(global_tempfile_name,0);
+ if(sp[-1]==DIRSEP) // it's a bare directory
+ strcpy(sp,"temp"); // add a file name prefix
+ strMcpy(tempfile_directory,global_tempfile_name);
+ sp= strrchr(tempfile_directory,DIRSEP);
+ // get last directory separator
+ if(sp!=NULL) *sp= 0; // if found any, cut the string here
+ #if __WIN32__
+ mkdir(tempfile_directory);
+ #else
+ mkdir(tempfile_directory,0700);
+ #endif
+ }
+
+ // get file timestamp of OSM input file
+ if(old_timestamp==0) { // no timestamp given by the user
+ if(old_file==NULL) { // no file name given for the old OSM file
+ PERR("Specify at least the old OSM file's name or its timestamp.")
+return 1;
+ }
+ if(!file_exists(old_file)) { // old OSM file does not exist
+ PERRv("Old OSM file does not exist: %.80s",old_file);
+return 1;
+ }
+ old_timestamp= get_file_timestamp(old_file);
+ if(old_timestamp==0) {
+ PERRv("Old OSM file does not contain a timestamp: %.80s",old_file);
+ PERR("Please specify the timestamp manually, e.g.: "
+ "2011-07-15T23:30:00Z");
+return 1;
+ }
+ } // end no timestamp given by the user
+
+ // parameter consistency check
+ if(new_file==NULL) {
+ PERR("No output file was specified.");
+return 1;
+ }
+ if(old_file!=NULL && strcmp(old_file,new_file)==0) {
+ PERR("Input file and output file are identical.");
+return 1;
+ }
+ if(old_file==NULL && !new_file_is_changefile) {
+ PERR("If no old OSM file is specified, osmupdate can only "
+ "generate a changefile.");
+return 1;
+ }
+
+ // initialize sequence numbers and timestamps
+ minutely_sequence_number= hourly_sequence_number=
+ daily_sequence_number= sporadic_sequence_number= 0;
+ minutely_timestamp= hourly_timestamp=
+ daily_timestamp= sporadic_timestamp= 0;
+
+ // care about user defined processing categories
+ if(process_minutely || process_hourly ||
+ process_daily || process_sporadic) {
+ // user wants specific type(s) of chancefiles to be processed
+ if(!process_minutely) no_minutely= true;
+ if(!process_hourly) no_hourly= true;
+ if(!process_daily) no_daily= true;
+ }
+ else {
+ // try to get sporadic timestamp
+ sporadic_timestamp= get_newest_changefile_timestamp(
+ cft_SPORADIC,&sporadic_sequence_number);
+ if(sporadic_timestamp!=0) {
+ // there is a timestamp at the highest directory level,
+ // this must be a so-called sporadic timestamp
+ if(loglevel>0) {
+ PINFO("Found status information in base URL root.")
+ PINFO("Ignoring subdirectories \"minute\", \"hour\", \"day\".")
+ }
+ process_sporadic= true; // let's take it
+ no_minutely= no_hourly= no_daily= true;
+ }
+ }
+
+ // get last timestamp for each, minutely, hourly, daily,
+ // and sporadic diff files
+ if(!no_minutely) {
+ minutely_timestamp= get_newest_changefile_timestamp(
+ cft_MINUTELY,&minutely_sequence_number);
+ if(minutely_timestamp==0) {
+ PERR("Could not get the newest minutely timestamp from the Internet.")
+return 1;
+ }
+ }
+ if(!no_hourly) {
+ hourly_timestamp= get_newest_changefile_timestamp(
+ cft_HOURLY,&hourly_sequence_number);
+ if(hourly_timestamp==0) {
+ PERR("Could not get the newest hourly timestamp from the Internet.")
+ return 1;
+ }
+ }
+ if(!no_daily) {
+ daily_timestamp= get_newest_changefile_timestamp(
+ cft_DAILY,&daily_sequence_number);
+ if(daily_timestamp==0) {
+ PERR("Could not get the newest daily timestamp from the Internet.")
+ return 1;
+ }
+ }
+ if(process_sporadic && sporadic_timestamp==0) {
+ sporadic_timestamp= get_newest_changefile_timestamp(
+ cft_SPORADIC,&sporadic_sequence_number);
+ if(sporadic_timestamp==0) {
+ PERR("Could not get the newest sporadic timestamp "
+ "from the Internet.")
+ return 1;
+ }
+ }
+
+ // check maximum update range
+ if(minutely_timestamp-old_timestamp>max_update_range) {
+ // update range too large
+ int days;
+ days= (int)((minutely_timestamp-old_timestamp+86399)/86400);
+ PERRv("Update range too large: %i days.",days)
+ PINFOv("To allow such a wide range, add: --max-days=%i",days)
+return 1;
+ } // update range too large
+
+ // clear last hourly timestamp if
+ // OSM old file's timestamp > latest hourly timestamp - 30 minutes
+ if(old_timestamp>hourly_timestamp-30*60 && !no_minutely)
+ hourly_timestamp= 0; // (let's take minutely updates)
+
+ // clear last daily timestamp if
+ // OSM file timestamp > latest daily timestamp - 16 hours
+ if(old_timestamp>daily_timestamp-16*3600 &&
+ !(no_hourly && no_minutely))
+ daily_timestamp= 0; // (let's take hourly and minutely updates)
+
+ // initialize start timestamp
+ timestamp= 0;
+ if(timestamphourly_timestamp &&
+ next_timestamp>old_timestamp) {
+ timestamp= next_timestamp;
+ process_changefile(cft_MINUTELY,minutely_sequence_number,timestamp);
+ minutely_sequence_number--;
+ next_timestamp= get_changefile_timestamp(
+ cft_MINUTELY,minutely_sequence_number);
+ }
+ }
+
+ // get and process hourly diff files from last hourly timestamp
+ // backward; stop just before last daily timestamp or OSM
+ // file timestamp has been reached;
+ if(hourly_timestamp!=0) {
+ next_timestamp= timestamp;
+ while(next_timestamp>daily_timestamp &&
+ next_timestamp>old_timestamp) {
+ timestamp= next_timestamp;
+ process_changefile(cft_HOURLY,hourly_sequence_number,timestamp);
+ hourly_sequence_number--;
+ next_timestamp= get_changefile_timestamp(
+ cft_HOURLY,hourly_sequence_number);
+ }
+ }
+
+ // get and process daily diff files from last daily timestamp
+ // backward; stop just before OSM file timestamp has been reached;
+ if(daily_timestamp!=0) {
+ next_timestamp= timestamp;
+ while(next_timestamp>old_timestamp) {
+ timestamp= next_timestamp;
+ process_changefile(cft_DAILY,daily_sequence_number,timestamp);
+ daily_sequence_number--;
+ next_timestamp= get_changefile_timestamp(
+ cft_DAILY,daily_sequence_number);
+ }
+ }
+
+ // get and process sporadic diff files from last sporadic timestamp
+ // backward; stop just before OSM file timestamp has been reached;
+ if(sporadic_timestamp!=0) {
+ next_timestamp= timestamp;
+ while(next_timestamp>old_timestamp) {
+ timestamp= next_timestamp;
+ process_changefile(
+ cft_SPORADIC,sporadic_sequence_number,timestamp);
+ sporadic_sequence_number--;
+ next_timestamp= get_changefile_timestamp(
+ cft_SPORADIC,sporadic_sequence_number);
+ }
+ }
+
+ // process remaining files which may still wait in the cache;
+ process_changefile(0,0,0);
+
+ /* create requested output file */ {
+ char master_cachefile_name[400];
+ char command[2000],*command_p;
+ char* command_e= command+sizeof(command);
+ char result[1000];
+
+ if(loglevel>0)
+ PINFO("Creating output file.")
+ strcpy(stpmcpy(master_cachefile_name,global_tempfile_name,
+ sizeof(master_cachefile_name)-5),".8");
+ if(!file_exists(master_cachefile_name)) {
+ if(old_file==NULL)
+ PINFO("There is no changefile since this timestamp.")
+ else
+ PINFO("Your OSM file is already up-to-date.")
+return 21;
+ }
+ command_p= command;
+ if(new_file_is_changefile) { // changefile
+ if(new_file_is_gz) { // compressed
+ if(new_file_is_o5) { // .o5c.gz
+ stecpy(&command_p,command_e,"gzip ");
+ stecpy(&command_p,command_e,global_gzip_parameters);
+ stecpy(&command_p,command_e," <\"");
+ steesccpy(&command_p,command_e,master_cachefile_name);
+ stecpy(&command_p,command_e,"\" >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ else { // .osc.gz
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," ");
+ stecpy(&command_p,command_e,global_osmconvert_arguments);
+ stecpy(&command_p,command_e," \"");
+ steesccpy(&command_p,command_e,master_cachefile_name);
+ stecpy(&command_p,command_e,"\" --out-osc |gzip ");
+ stecpy(&command_p,command_e,global_gzip_parameters);
+ stecpy(&command_p,command_e," >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ shell_command(command,result);
+ } // compressed
+ else { // uncompressed
+ if(new_file_is_o5) // .o5c
+ rename(master_cachefile_name,new_file);
+ else { // .osc
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," ");
+ stecpy(&command_p,command_e,global_osmconvert_arguments);
+ stecpy(&command_p,command_e," \"");
+ steesccpy(&command_p,command_e,master_cachefile_name);
+ stecpy(&command_p,command_e,"\" --out-osc >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ shell_command(command,result);
+ }
+ } // uncompressed
+ } // changefile
+ else { // OSM file
+ #if 0
+ if(loglevel>=2) {
+ PINFOv("oc %s",global_osmconvert_program)
+ PINFOv("fa %s",final_osmconvert_arguments)
+ PINFOv("of %s",old_file)
+ PINFOv("mc %s",master_cachefile_name)
+ }
+ #endif
+ stecpy(&command_p,command_e,global_osmconvert_program);
+ stecpy(&command_p,command_e," ");
+ stecpy(&command_p,command_e,final_osmconvert_arguments);
+ stecpy(&command_p,command_e," \"");
+ steesccpy(&command_p,command_e,old_file);
+ stecpy(&command_p,command_e,"\" \"");
+ steesccpy(&command_p,command_e,master_cachefile_name);
+ if(new_file_is_gz) { // compressed
+ if(new_file_is_o5) { // .o5m.gz
+ stecpy(&command_p,command_e,"\" --out-o5m |gzip ");
+ stecpy(&command_p,command_e,global_gzip_parameters);
+ stecpy(&command_p,command_e," >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ else { // .osm.gz
+ stecpy(&command_p,command_e,"\" --out-osm |gzip ");
+ stecpy(&command_p,command_e,global_gzip_parameters);
+ stecpy(&command_p,command_e," >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ } // compressed
+ else { // uncompressed
+ if(new_file_is_pbf) { // .pbf
+ stecpy(&command_p,command_e,"\" --out-pbf >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ else if(new_file_is_o5) { // .o5m
+ stecpy(&command_p,command_e,"\" --out-o5m >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ else { // .osm
+ stecpy(&command_p,command_e,"\" --out-osm >\"");
+ steesccpy(&command_p,command_e,new_file);
+ stecpy(&command_p,command_e,"\"");
+ }
+ } // uncompressed
+ shell_command(command,result);
+ } // OSM file
+ if(loglevel<2)
+ unlink(master_cachefile_name);
+ } // create requested output file
+
+ // delete tempfiles
+ if(global_keep_tempfiles) { // tempfiles shall be kept
+ if(loglevel>0)
+ PINFO("Keeping temporary files.")
+ } // tempfiles shall be kept
+ else { // tempfiles shall be deleted
+ char command[500],*command_p,result[1000];
+ char* command_e= command+sizeof(command);
+
+ if(loglevel>0)
+ PINFO("Deleting temporary files.")
+ command_p= command;
+ stecpy(&command_p,command_e,DELFILE" \"");
+ steesccpy(&command_p,command_e,global_tempfile_name);
+ stecpy(&command_p,command_e,"\".*");
+ shell_command(command,result);
+ rmdir(tempfile_directory);
+ } // tempfiles shall be deleted
+
+ if(main_return_value==0 && loglevel>0)
+ PINFO("Completed successfully.")
+
+ return main_return_value;
+ } // end main()
+
diff --git a/docker-osmupdate/readme.md b/docker-osmupdate/readme.md
new file mode 100644
index 0000000..83bb263
--- /dev/null
+++ b/docker-osmupdate/readme.md
@@ -0,0 +1,20 @@
+## Docker OSM Update
+
+You should have 3 folders : base_pbf, import_queue, import_done
+
+Put a state file in base-pbf like this one :
+http://download.openstreetmap.fr/extracts/africa/south_africa.state.txt
+
+``docker build -t osmupdate .``
+``docker run -v $('pwd')import-queue/:/home/import-queue -v $('pwd')base-pbf/:/home/base-pbf -v $('pwd')import-done/:/home/import-done -d osmupdate``
+
+With -e, you can add some settings :
+ - MAX_DAYS = 100, the maximum time range to assamble a cumulated changefile.
+ - DIFF = sporadic, osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+ - MAX_MERGE = 7, argument to determine the maximum number of parallely processed changefiles.
+ - COMPRESSION_LEVEL = 1, define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
+ - BASE_URL = http://planet.openstreetmap.org/replication/, change the URL to use a custom URL to fetch regional file updates.
+ - IMPORT_QUEUE = import_queue
+ - IMPORT_DONE = import_done
+ - BASE_PBF = base_pbf
+ - TIME = 120, secondes between two executions of the script
\ No newline at end of file
diff --git a/settings/post-pbf-import.sql b/settings/post-pbf-import.sql
new file mode 100644
index 0000000..e69de29