Several bugfixes and improvements

ogn/commands/bulkimport.py

@@ -1,10 +1,12 @@
 import os
 import re
+import logging
 
 from manager import Manager
 from ogn.commands.dbutils import session
 from ogn.model import AircraftBeacon, ReceiverBeacon
 from ogn.utils import open_file
+from ogn.gateway.process_tools import FileSaver, Converter, Merger
 
 
 manager = Manager()
@@ -13,118 +15,101 @@ PATTERN = '^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$'
 
 
 @manager.command
-def convert_logfile(path, logfile='main.log', loglevel='INFO'):
+def convert_logfile(path):
     """Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
 
+    logging.basicConfig(filename='convert.log', level=logging.DEBUG)
+
     if os.path.isfile(path):
         head, tail = os.path.split(path)
         convert(tail, path=head)
-        print("Finished")
+        logging.info("Finished converting single file {}".format(head))
     elif os.path.isdir(path):
         for filename in os.listdir(path):
             convert(filename, path=path)
-        print("Finished")
+        logging.info("Finished converting file path {}".format(path))
     else:
-        print("Not a file nor a path: {}".format(path))
+        logging.warning("Not a file nor a path: {}".format(path))
 
 
 def convert(sourcefile, path=''):
-    import csv
+    logging.info("convert: {} {}".format(sourcefile, path))
-    import gzip
     import datetime
 
-    from ogn.gateway.process import message_to_beacon
+    from ogn.gateway.process import string_to_message
 
     match = re.search(PATTERN, sourcefile)
     if match:
         reference_date_string = match.group(1)
         reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d")
 
-        aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
+        # Build the processing pipeline
-        receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
+        saver = FileSaver()
+        converter = Converter(callback=saver)
+        merger = Merger(callback=converter)
 
-        if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
+        try:
-            print("Reading file: {}".format(sourcefile))
+            saver.open(path, reference_date_string)
-            fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
+        except FileExistsError:
-            fout_rb = gzip.open(receiver_beacon_filename, 'wt')
+            logging.warning("Output files already exists. Skipping")
-        else:
-            print("Output files for file {} already exists. Skipping".format(sourcefile))
             return
     else:
-        print("filename '{}' does not match pattern. Skipping".format(sourcefile))
+        logging.warning("filename '{}' does not match pattern. Skipping".format(sourcefile))
         return
 
     fin = open_file(os.path.join(path, sourcefile))
 
     # get total lines of the input file
-    total = 0
+    total_lines = 0
     for line in fin:
-        total += 1
+        total_lines += 1
     fin.seek(0)
 
-    aircraft_beacons = list()
-    receiver_beacons = list()
-
     progress = -1
-    num_lines = 0
+    current_line = 0
-
-    wr_ab = csv.writer(fout_ab, delimiter=',')
-    wr_ab.writerow(AircraftBeacon.get_csv_columns())
-
-    wr_rb = csv.writer(fout_rb, delimiter=',')
-    wr_rb.writerow(ReceiverBeacon.get_csv_columns())
 
     print('Start importing ogn-logfile')
     for line in fin:
-        num_lines += 1
+        current_line += 1
-        if int(100 * num_lines / total) != progress:
+        if int(1000 * current_line / total_lines) != progress:
-            progress = round(100 * num_lines / total)
+            progress = round(1000 * current_line / total_lines)
-            print("\rReading line {} ({}%)".format(num_lines, progress), end='')
+            print("\rReading line {} ({}%)".format(current_line, progress / 10), end='')
-            if len(aircraft_beacons) > 0:
-                for beacon in aircraft_beacons:
-                    wr_ab.writerow(beacon.get_csv_values())
-                aircraft_beacons = list()
-            if len(receiver_beacons) > 0:
-                for beacon in receiver_beacons:
-                    wr_rb.writerow(beacon.get_csv_values())
-                receiver_beacons = list()
 
-        beacon = message_to_beacon(line.strip(), reference_date=reference_date, wait_for_brother=True)
+        message = string_to_message(line.strip(), reference_date=reference_date)
-        if beacon is not None:
+        if message is None:
-            if isinstance(beacon, AircraftBeacon):
+            print("=====")
-                aircraft_beacons.append(beacon)
+            print(line.strip())
-            elif isinstance(beacon, ReceiverBeacon):
+            continue
-                receiver_beacons.append(beacon)
+        
+        merger.add_message(message)
 
-    if len(aircraft_beacons) > 0:
+    merger.flush()
-        for beacon in aircraft_beacons:
+    saver.close()
-            wr_ab.writerow(beacon.get_csv_values())
-    if len(receiver_beacons) > 0:
-        for beacon in receiver_beacons:
-            wr_rb.writerow(beacon.get_csv_values())
 
     fin.close()
-    fout_ab.close()
-    fout_rb.close()
 
 
 @manager.command
 def drop_indices():
     """Drop indices of AircraftBeacon."""
     session.execute("""
-        DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name;
+        DROP INDEX IF EXISTS idx_aircraft_beacons_location;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_date_device_id_address;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_date_receiver_id_distance;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_location;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs;
         DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp;
+        
+        DROP INDEX IF EXISTS idx_receiver_beacons_location;
+        DROP INDEX IF EXISTS ix_receiver_beacons_date_receiver_id;
+        DROP INDEX IF EXISTS ix_receiver_beacons_timestamp;
     """)
-    print("Dropped indices of AircraftBeacon")
+    print("Dropped indices of AircraftBeacon and ReceiverBeacon")
 
     # disable constraint trigger
     session.execute("""
-        ALTER TABLE aircraft_beacons DISABLE TRIGGER ALL
+        ALTER TABLE aircraft_beacons DISABLE TRIGGER ALL;
+        ALTER TABLE receiver_beacons DISABLE TRIGGER ALL;
     """)
+    session.commit()
     print("Disabled constraint triggers")
 
 
@@ -132,18 +117,22 @@ def drop_indices():
 def create_indices():
     """Create indices for AircraftBeacon."""
     session.execute("""
-        CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name);
+        CREATE INDEX idx_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
-        CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address);
+        CREATE INDEX ix_aircraft_beacons_date_device_id_address ON aircraft_beacons USING BTREE((timestamp::date), device_id, address);
-        CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp);
+        CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING BTREE((timestamp::date), receiver_id, distance);
-        CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
+        CREATE INDEX ix_aircraft_beacons_timestamp ON aircraft_beacons USING BTREE(timestamp);
-
+        
-        CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance)
+        CREATE INDEX idx_receiver_beacons_location ON receiver_beacons USING GIST(location);
+        CREATE INDEX ix_receiver_beacons_date_receiver_id ON receiver_beacons USING BTREE((timestamp::date), receiver_id);
+        CREATE INDEX ix_receiver_beacons_timestamp ON receiver_beacons USING BTREE(timestamp);
     """)
-    print("Created indices for AircraftBeacon")
+    print("Created indices for AircraftBeacon and ReceiverBeacon")
 
     session.execute("""
-        ALTER TABLE aircraft_beacons ENABLE TRIGGER ALL
+        ALTER TABLE aircraft_beacons ENABLE TRIGGER ALL;
+        ALTER TABLE receiver_beacons ENABLE TRIGGER ALL;
     """)
+    session.commit()
     print("Enabled constraint triggers")
 
 
@@ -198,6 +187,11 @@ def import_logfile(path):
         else:
             print("For {} beacons already exist. Skipping".format(reference_date_string))
     else:
+        s1 = header
+        s2 = ','.join(AircraftBeacon.get_csv_columns())
+        print(s1)
+        print(s2)
+        print([i for i in range(len(s1)) if s1[i] != s2[i]])
         print("Unknown file type: {}".format(tail))
 
 
@@ -214,32 +208,34 @@ def import_aircraft_beacon_logfile(csv_logfile):
     DROP TABLE IF EXISTS aircraft_beacons_temp;
     CREATE TABLE aircraft_beacons_temp(
         location geometry,
-        altitude integer,
+        altitude real,
         name character varying,
         dstcall character varying,
         relay character varying,
         receiver_name character varying(9),
         "timestamp" timestamp without time zone,
-        track integer,
+        track smallint,
-        ground_speed double precision,
+        ground_speed real,
 
         address_type smallint,
         aircraft_type smallint,
         stealth boolean,
-        address character varying(6),
+        address character varying,
-        climb_rate double precision,
+        climb_rate real,
-        turn_rate double precision,
+        turn_rate real,
-        flightlevel double precision,
+        signal_quality real,
-        signal_quality double precision,
+        error_count smallint,
-        error_count integer,
+        frequency_offset real,
-        frequency_offset double precision,
+        gps_quality_horizontal smallint,
-        gps_status character varying,
+        gps_quality_vertical smallint,
-        software_version double precision,
+        software_version real,
         hardware_version smallint,
         real_address character varying(6),
-        signal_power double precision,
+        signal_power real,
 
-        distance double precision,
+        distance real,
+        radial smallint,
+        normalized_signal_quality real,
         location_mgrs character varying(15)
         );
     """
@@ -287,10 +283,12 @@ def import_aircraft_beacon_logfile(csv_logfile):
 
     session.execute("""
         INSERT INTO aircraft_beacons(location, altitude, name, dstcall, relay, receiver_name, timestamp, track, ground_speed,
-                                    address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power, distance, location_mgrs,
+                                    address_type, aircraft_type, stealth, address, climb_rate, turn_rate, signal_quality, error_count, frequency_offset, gps_quality_horizontal, gps_quality_vertical, software_version, hardware_version, real_address, signal_power,
+                                    distance, radial, normalized_signal_quality, location_mgrs,
                                     receiver_id, device_id)
         SELECT t.location, t.altitude, t.name, t.dstcall, t.relay, t.receiver_name, t.timestamp, t.track, t.ground_speed,
-               t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power, t.distance, t.location_mgrs,
+               t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.signal_quality, t.error_count, t.frequency_offset, t.gps_quality_horizontal, t.gps_quality_vertical, t.software_version, t.hardware_version, t.real_address, t.signal_power,
+               t.distance, t.radial, t.normalized_signal_quality, t.location_mgrs,
                r.id, d.id
         FROM aircraft_beacons_temp t, receivers r, devices d
         WHERE t.receiver_name = r.name AND t.address = d.address
@@ -311,7 +309,7 @@ def import_receiver_beacon_logfile(csv_logfile):
     DROP TABLE IF EXISTS receiver_beacons_temp;
     CREATE TABLE receiver_beacons_temp(
         location geometry,
-        altitude integer,
+        altitude real,
         name character varying,
         receiver_name character varying(9),
         dstcall character varying,
@@ -319,20 +317,20 @@ def import_receiver_beacon_logfile(csv_logfile):
 
         version character varying,
         platform character varying,
-        cpu_load double precision,
+        cpu_load real,
-        free_ram double precision,
+        free_ram real,
-        total_ram double precision,
+        total_ram real,
-        ntp_error double precision,
+        ntp_error real,
-        rt_crystal_correction double precision,
+        rt_crystal_correction real,
-        voltage double precision,
+        voltage real,
-        amperage double precision,
+        amperage real,
-        cpu_temp double precision,
+        cpu_temp real,
         senders_visible integer,
         senders_total integer,
-        rec_input_noise double precision,
+        rec_input_noise real,
-        senders_signal double precision,
+        senders_signal real,
         senders_messages integer,
-        good_senders_signal double precision,
+        good_senders_signal real,
         good_senders integer,
         good_and_bad_senders integer
         );

ogn/commands/database.py

@@ -22,8 +22,8 @@ def init():
     session.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
     session.commit()
     Base.metadata.create_all(engine)
-    alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
+    #alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
-    command.stamp(alembic_cfg, "head")
+    #command.stamp(alembic_cfg, "head")
     print("Done.")
 
 

ogn/gateway/process.py

@@ -1,24 +1,30 @@
 import logging
+from math import log10
 
 from mgrs import MGRS
-from haversine import haversine
 
+from ogn.utils import haversine
 from ogn.commands.dbutils import session
 from ogn.model import AircraftBeacon, ReceiverBeacon, Location
 from ogn.parser import parse, ParseError
-from datetime import datetime, timedelta
+from ogn.gateway.process_tools import DbSaver, Converter, DummyMerger, AIRCRAFT_TYPES, RECEIVER_TYPES
 
 
 logger = logging.getLogger(__name__)
 myMGRS = MGRS()
 
 
-def replace_lonlat_with_wkt(message, reference_receiver=None):
+
+def _replace_lonlat_with_wkt(message, reference_receiver=None):
     latitude = message['latitude']
     longitude = message['longitude']
 
     if reference_receiver is not None:
-        message['distance'] = 1000.0 * haversine((reference_receiver['latitude'], reference_receiver['longitude']), (latitude, longitude))
+        distance,bearing = haversine(reference_receiver['latitude'], reference_receiver['longitude'], latitude, longitude)
+        message['distance'] = distance
+        message['radial'] = round(bearing)
+        if message['signal_quality'] is not None and distance >= 1:
+            message['normalized_signal_quality'] = message['signal_quality'] + 20 * log10(message['distance'] / 10000)  # normalized to 10km
 
     location = Location(longitude, latitude)
     message['location_wkt'] = location.to_wkt()
@@ -27,88 +33,55 @@ def replace_lonlat_with_wkt(message, reference_receiver=None):
     del message['longitude']
     return message
 
-previous_message = None
+
 receivers = dict()
 
-
+def string_to_message(raw_string, reference_date):
-def message_to_beacon(raw_message, reference_date, wait_for_brother=False):
-    beacon = None
-    global previous_message
     global receivers
 
-    if raw_message[0] != '#':
+    try:
-        try:
+        message = parse(raw_string, reference_date)
-            message = parse(raw_message, reference_date)
+    except NotImplementedError as e:
-        except NotImplementedError as e:
+        logger.error('No parser implemented for message: {}'.format(raw_string))
-            logger.error('Received message: {}'.format(raw_message))
+        return None
-            logger.error(e)
+    except ParseError as e:
-            return None
+        logger.error('Parsing error with message: {}'.format(raw_string))
-        except ParseError as e:
+        return None
-            logger.error('Received message: {}'.format(raw_message))
+    except TypeError as e:
-            logger.error('Drop packet, {}'.format(e.message))
+        logger.error('TypeError with message: {}'.format(raw_string))
-            return None
+        return None
-        except TypeError as e:
+    except Exception as e:
-            logger.error('TypeError: {}'.format(raw_message))
+        logger.error(raw_string)
-            return None
+        logger.error(e)
-        except Exception as e:
+        return None
-            logger.error(raw_message)
-            logger.error(e)
-            return None
 
-        # update reference receivers and distance to the receiver
+    # update reference receivers and distance to the receiver
-        if message['aprs_type'] == 'position':
+    if message['aprs_type'] == 'position':
-            if message['beacon_type'] in ['receiver_beacon', 'aprs_receiver', 'receiver']:
+        if message['beacon_type'] in RECEIVER_TYPES:
-                receivers.update({message['name']: {'latitude': message['latitude'], 'longitude': message['longitude']}})
+            receivers.update({message['name']: {'latitude': message['latitude'], 'longitude': message['longitude']}})
-                message = replace_lonlat_with_wkt(message)
+            message = _replace_lonlat_with_wkt(message)
-            elif message['beacon_type'] in ['aircraft_beacon', 'aprs_aircraft', 'flarm', 'tracker']:
+        elif message['beacon_type'] in AIRCRAFT_TYPES:
-                reference_receiver = receivers.get(message['receiver_name'])
+            reference_receiver = receivers.get(message['receiver_name'])
-                message = replace_lonlat_with_wkt(message, reference_receiver=reference_receiver)
+            message = _replace_lonlat_with_wkt(message, reference_receiver=reference_receiver)
+            if 'gps_quality' in message and message['gps_quality'] is not None and 'horizontal' in message['gps_quality']:
+                message['gps_quality_horizontal'] = message['gps_quality']['horizontal']
+                message['gps_quality_vertical'] = message['gps_quality']['vertical']
 
-        # optional: merge different beacon types
+    # update raw_message
-        params = dict()
+    message['raw_message'] = raw_string
-        if wait_for_brother is True:
-            if previous_message is None:
-                previous_message = message
-                return None
-            elif message['name'] == previous_message['name'] and message['timestamp'] == previous_message['timestamp']:
-                params = message
-                params.update(previous_message)
-                params['aprs_type'] = 'merged'
-                previous_message = None
-            else:
-                params = previous_message
-                previous_message = message
-        else:
-            params = message
 
-        # create beacons
+    return message
-        if params['beacon_type'] in ['aircraft_beacon', 'aprs_aircraft', 'flarm', 'tracker']:
-            beacon = AircraftBeacon(**params)
-        elif params['beacon_type'] in ['receiver_beacon', 'aprs_receiver', 'receiver']:
-            beacon = ReceiverBeacon(**params)
-        else:
-            print("Whoops: what is this: {}".format(params))
-
-    return beacon
-
-beacons = list()
-last_commit = datetime.utcnow()
 
 
-def process_beacon(raw_message, reference_date=None):
+# Build the processing pipeline
-    global beacons
+saver = DbSaver(session=session)
-    global last_commit
+converter = Converter(callback=saver)
+merger = DummyMerger(callback=converter)
+
+
+def process_raw_message(raw_message, reference_date=None, merger=merger):
+    logger.debug('Received message: {}'.format(raw_message))
+    message = string_to_message(raw_message, reference_date)
+    merger.add_message(message)
 
-    beacon = message_to_beacon(raw_message, reference_date)
-    if beacon is not None:
-        beacons.append(beacon)
-        logger.debug('Received message: {}'.format(raw_message))
 
-    current_time = datetime.utcnow()
-    elapsed_time = current_time - last_commit
-    if elapsed_time >= timedelta(seconds=1):
-        session.bulk_save_objects(beacons)
-        session.commit()
-        logger.debug('Commited beacons')
-        beacons = list()
-        last_commit = current_time

ogn/gateway/process_tools.py

@@ -0,0 +1,184 @@
+from datetime import datetime, timedelta
+from ogn.model import AircraftBeacon, ReceiverBeacon
+
+AIRCRAFT_TYPES = ['aprs_aircraft', 'flarm', 'tracker', 'fanet', 'lt24', 'naviter', 'skylines', 'spider', 'spot']
+RECEIVER_TYPES = ['aprs_receiver', 'receiver']
+
+class DummyMerger:
+    def __init__(self, callback):
+        self.callback = callback
+
+    def add_message(self, message):
+        self.callback.add_message(message)
+
+    def flush(self):
+        pass
+
+class Merger:
+    def __init__(self, callback, max_timedelta=None, max_lines=None):
+        self.callback = callback
+        self.max_timedelta = max_timedelta
+        self.max_lines = max_lines
+        self.message_map = dict()
+
+    def add_message(self, message):
+        if message is None or ('raw_message' in message and message['raw_message'][0] == '#'):
+            return
+
+        # release old messages
+        if self.max_timedelta is not None:
+            for receiver,v1 in self.message_map.items():
+                for name,v2 in v1.items():
+                    for timestamp,message in v2.items():
+                        if message['timestamp'] - timestamp > self.max_timedelta:
+                            self.callback.add_message(message)
+                            del self.message_map[receiver][name][timestamp]
+
+        # release messages > max_lines
+        if self.max_lines is not None:
+            pass
+
+        # merge messages with same timestamp
+        if message['receiver_name'] in self.message_map:
+            if message['name'] in self.message_map[message['receiver_name']]:
+                messages = self.message_map[message['receiver_name']][message['name']]
+                if message['timestamp'] in messages:
+                    other = messages[message['timestamp']]
+                    params1 = dict( [(k,v) for k,v in message.items() if v is not None])
+                    params2 = dict( [(k,v) for k,v in other.items() if v is not None])
+                    merged = {**params1, **params2}
+                    
+                    # zum debuggen
+                    if 'raw_message' in message and 'raw_message' in other:
+                        merged['raw_message'] = '"{}","{}"'.format(message['raw_message'], other['raw_message'])
+                    
+                    self.callback.add_message(merged)
+                    del self.message_map[message['receiver_name']][message['name']][message['timestamp']]
+                else:
+                    self.message_map[message['receiver_name']][message['name']][message['timestamp']] = message
+    
+                # release previous messages
+                for timestamp in list(messages):
+                    if timestamp < message['timestamp']:
+                        self.callback.add_message(messages[timestamp])
+                        del self.message_map[message['receiver_name']][message['name']][timestamp]
+            else:
+                # add new message
+                self.message_map[message['receiver_name']].update({message['name']: {message['timestamp']: message}})
+        else:
+            self.message_map.update({message['receiver_name']: {message['name']: {message['timestamp']: message}}})
+
+    def flush(self):
+        for receiver,v1 in self.message_map.items():
+            for name,v2 in v1.items():
+                for timestamp in v2:
+                    self.callback.add_message(self.message_map[receiver][name][timestamp])
+
+        self.callback.flush()
+        self.message_map = dict()
+
+class Converter:
+    def __init__(self, callback):
+        self.callback = callback
+    
+    def add_message(self, message):
+        try:
+            beacon = self.message_to_beacon(message)
+            self.callback.add_message(beacon)
+        except Exception as e:
+            print(e)
+            print(message)
+            return
+
+    def message_to_beacon(self, message):
+        # create beacons
+        if message['beacon_type'] in AIRCRAFT_TYPES:
+            beacon = AircraftBeacon(**message)
+        elif message['beacon_type'] in RECEIVER_TYPES:
+            if 'rec_crystal_correction' in message:
+                del message['rec_crystal_correction']
+                del message['rec_crystal_correction_fine']
+            beacon = ReceiverBeacon(**message)
+        else:
+            print("Whoops: what is this: {}".format(message))
+    
+        return beacon
+    
+    def flush(self):
+        self.callback.flush()
+
+class DummySaver:
+    def add_message(self, message):
+        if message['beacon_type'] != "aprs_aircraft":
+            print(message['beacon_type'])
+
+    def flush(self):
+        print("========== flush ==========")
+
+class DbSaver:
+    def __init__(self, session):
+        self.session = session
+        self.beacons = list()
+        self.last_commit = datetime.utcnow()
+
+    def add_message(self, beacon):
+        global last_commit
+        global beacons
+
+        self.beacons.append(beacon)
+
+        elapsed_time = datetime.utcnow() - self.last_commit
+        if elapsed_time >= timedelta(seconds=1):
+            self.flush()
+
+    def flush(self):
+        try:
+            self.session.bulk_save_objects(self.beacons)
+            self.session.commit()
+            self.beacons = list()
+            self.last_commit = datetime.utcnow()
+        except Exception as e:
+            self.session.rollback()
+            print(e)
+            return
+
+import os, gzip, csv
+
+class FileSaver:
+    def __init__(self):
+        self.aircraft_messages = list()
+        self.receiver_messages = list()
+
+    def open(self, path, reference_date_string):
+        aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
+        receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
+
+        if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
+            self.fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
+            self.fout_rb = gzip.open(receiver_beacon_filename, 'wt')
+        else:
+            raise FileExistsError
+        
+        self.aircraft_writer = csv.writer(self.fout_ab, delimiter=',')
+        self.aircraft_writer.writerow(AircraftBeacon.get_csv_columns())
+
+        self.receiver_writer = csv.writer(self.fout_rb, delimiter=',')
+        self.receiver_writer.writerow(ReceiverBeacon.get_csv_columns())
+        
+        return 1
+
+    def add_message(self, beacon):
+        if isinstance(beacon, AircraftBeacon):
+            self.aircraft_messages.append(beacon.get_csv_values())
+        elif isinstance(beacon, ReceiverBeacon):
+            self.receiver_messages.append(beacon.get_csv_values())
+
+    def flush(self):
+        self.aircraft_writer.writerows(self.aircraft_messages)
+        self.receiver_writer.writerows(self.receiver_messages)
+        self.aircraft_messages = list()
+        self.receiver_messages = list()
+        
+    def close(self):
+        self.fout_ab.close()
+        self.fout_rb.close()

ogn/model/aircraft_beacon.py

@@ -7,28 +7,28 @@ from .beacon import Beacon
 class AircraftBeacon(Beacon):
     __tablename__ = "aircraft_beacons"
 
-    # Activate relay for AircraftBeacon
-    relay = Column(String)
-
     # Flarm specific data
     address_type = Column(SmallInteger)
     aircraft_type = Column(SmallInteger)
     stealth = Column(Boolean)
-    address = Column(String(6))
+    address = Column(String)
     climb_rate = Column(Float(precision=2))
     turn_rate = Column(Float(precision=2))
-    flightlevel = Column(Float(precision=2))
     signal_quality = Column(Float(precision=2))
     error_count = Column(SmallInteger)
     frequency_offset = Column(Float(precision=2))
-    gps_status = Column(String)
+    gps_quality_horizontal = Column(SmallInteger)
+    gps_quality_vertical = Column(SmallInteger)
     software_version = Column(Float(precision=2))
     hardware_version = Column(SmallInteger)
     real_address = Column(String(6))
     signal_power = Column(Float(precision=2))
-
-    # Not so very important data
     proximity = None
+    
+    # Tracker stuff (position message)
+    flightlevel = None
+
+    # Tracker stuff (status message)
     gps_satellites = None
     gps_quality = None
     gps_altitude = None
@@ -40,10 +40,16 @@ class AircraftBeacon(Beacon):
     noise_level = None
     relays = None
 
+    # Spider stuff
+    spider_id = None
+    model = None
+    status = None
+
     # Calculated values
-    status = Column(SmallInteger, default=0)
+    distance = Column(Float(precision=2))
-    distance = Column(Float)
+    radial = Column(SmallInteger)
-    location_mgrs = Column(String(15), index=True)
+    normalized_signal_quality = Column(Float(precision=2))
+    location_mgrs = Column(String(15))
 
     # Relations
     receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'))
@@ -58,24 +64,26 @@ class AircraftBeacon(Beacon):
     Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
 
     def __repr__(self):
-        return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+        return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
             self.address_type,
             self.aircraft_type,
             self.stealth,
             self.address,
             self.climb_rate,
             self.turn_rate,
-            self.flightlevel,
             self.signal_quality,
             self.error_count,
             self.frequency_offset,
-            self.gps_status,
+            self.gps_quality_horizontal,
+            self.gps_quality_vertical,
             self.software_version,
             self.hardware_version,
             self.real_address,
             self.signal_power,
 
             self.distance,
+            self.radial,
+            self.normalized_signal_quality,
             self.location_mgrs)
 
     @classmethod
@@ -89,6 +97,9 @@ class AircraftBeacon(Beacon):
                'timestamp',
                'track',
                'ground_speed',
+               
+               #'raw_message',
+               #'reference_timestamp',
 
                'address_type',
                'aircraft_type',
@@ -96,23 +107,25 @@ class AircraftBeacon(Beacon):
                'address',
                'climb_rate',
                'turn_rate',
-               'flightlevel',
                'signal_quality',
                'error_count',
                'frequency_offset',
-               'gps_status',
+               'gps_quality_horizontal',
+               'gps_quality_vertical',
                'software_version',
                'hardware_version',
                'real_address',
                'signal_power',
 
                'distance',
+               'radial',
+               'normalized_signal_quality',
                'location_mgrs']
 
     def get_csv_values(self):
         return [
             self.location_wkt,
-            int(self.altitude),
+            int(self.altitude) if self.altitude else None,
             self.name,
             self.dstcall,
             self.relay,
@@ -120,6 +133,9 @@ class AircraftBeacon(Beacon):
             self.timestamp,
             self.track,
             self.ground_speed,
+            
+            #self.raw_message,
+            #self.reference_timestamp,
 
             self.address_type,
             self.aircraft_type,
@@ -127,18 +143,21 @@ class AircraftBeacon(Beacon):
             self.address,
             self.climb_rate,
             self.turn_rate,
-            self.flightlevel,
             self.signal_quality,
             self.error_count,
             self.frequency_offset,
-            self.gps_status,
+            self.gps_quality_horizontal,
+            self.gps_quality_vertical,
             self.software_version,
             self.hardware_version,
             self.real_address,
             self.signal_power,
 
             self.distance,
+            self.radial,
+            self.normalized_signal_quality,
             self.location_mgrs]
 
 
+Index('ix_aircraft_beacons_date_device_id_address', func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address)
 Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance)

ogn/model/beacon.py

@@ -16,7 +16,7 @@ class Beacon(AbstractConcreteBase, Base):
 
     name = Column(String)
     dstcall = Column(String)
-    relay = None
+    relay = Column(String)
     receiver_name = Column(String(9))
     timestamp = Column(DateTime, index=True)
     symboltable = None
@@ -25,10 +25,13 @@ class Beacon(AbstractConcreteBase, Base):
     ground_speed = Column(Float(precision=2))
     comment = None
 
+    # Type information
     beacon_type = None
     aprs_type = None
 
-    location_mgrs = None
+    # Debug information
+    raw_message = None #Column(String)
+    reference_timestamp = None #Column(DateTime, index=True)
 
     @property
     def location(self):

ogn/model/device.py

@@ -9,7 +9,8 @@ class Device(Base):
 
     id = Column(Integer, primary_key=True)
 
-    address = Column(String(6), index=True)
+    #address = Column(String(6), index=True)
+    address = Column(String, index=True)
     firstseen = Column(DateTime, index=True)
     lastseen = Column(DateTime, index=True)
     aircraft_type = Column(SmallInteger, index=True)

ogn/model/device_info.py

@@ -9,7 +9,8 @@ class DeviceInfo(Base):
 
     id = Column(Integer, primary_key=True)
     address_type = None
-    address = Column(String(6), index=True)
+    #address = Column(String(6), index=True)
+    address = Column(String, index=True)
     aircraft = Column(String)
     registration = Column(String(7))
     competition = Column(String(3))

ogn/model/device_stats.py

@@ -11,10 +11,7 @@ class DeviceStats(Base):
 
     date = Column(Date)
 
-    # Statistic data
+    # Static data
-    max_altitude = Column(Float(precision=2))
-    receiver_count = Column(SmallInteger)
-    aircraft_beacon_count = Column(Integer)
     firstseen = Column(DateTime)
     lastseen = Column(DateTime)
     aircraft_type = Column(SmallInteger)
@@ -23,7 +20,20 @@ class DeviceStats(Base):
     hardware_version = Column(SmallInteger)
     real_address = Column(String(6))
 
+    # Statistic data
+    max_altitude = Column(Float(precision=2))
+    receiver_count = Column(SmallInteger)
+    aircraft_beacon_count = Column(Integer)
+
     ambiguous = Column(Boolean)
+    
+    # Ranking data
+    max_altitude_ranking_worldwide = Column(Integer)
+    max_altitude_ranking_country = Column(Integer)
+    receiver_count_ranking_worldwide = Column(Integer)
+    receiver_count_ranking_country = Column(Integer)
+    aircraft_beacon_count_ranking_worldwide = Column(Integer)
+    aircraft_beacon_count_ranking_country = Column(Integer)
 
     # Relations
     device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)

ogn/model/receiver_beacon.py

@@ -1,13 +1,13 @@
 from sqlalchemy import Column, Float, String, Integer, SmallInteger, ForeignKey, Index
 from sqlalchemy.orm import relationship
-
+from sqlalchemy.sql import func
 from .beacon import Beacon
 
 
 class ReceiverBeacon(Beacon):
     __tablename__ = "receiver_beacons"
 
-    # disable not so important aprs fields
+    # disable irrelevant aprs fields
     track = None
     ground_speed = None
 
@@ -24,8 +24,6 @@ class ReceiverBeacon(Beacon):
     cpu_temp = Column(Float(precision=2))
     senders_visible = Column(Integer)
     senders_total = Column(Integer)
-    rec_crystal_correction = 0       # obsolete since 0.2.0
-    rec_crystal_correction_fine = 0  # obsolete since 0.2.0
     rec_input_noise = Column(Float(precision=2))
     senders_signal = Column(Float(precision=2))
     senders_messages = Column(Integer)
@@ -33,6 +31,7 @@ class ReceiverBeacon(Beacon):
     good_senders = Column(Integer)
     good_and_bad_senders = Column(Integer)
 
+    # User comment: used for additional information like hardware configuration, web site, email address, ...
     user_comment = None
 
     # Relations
@@ -43,7 +42,7 @@ class ReceiverBeacon(Beacon):
     Index('ix_receiver_beacons_receiver_id_name', 'receiver_id', 'name')
 
     def __repr__(self):
-        return "<ReceiverBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+        return "<ReceiverBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
             self.version,
             self.platform,
             self.cpu_load,
@@ -56,8 +55,6 @@ class ReceiverBeacon(Beacon):
             self.cpu_temp,
             self.senders_visible,
             self.senders_total,
-            # self.rec_crystal_correction,
-            # self.rec_crystal_correction_fine,
             self.rec_input_noise,
             self.senders_signal,
             self.senders_messages,
@@ -73,6 +70,9 @@ class ReceiverBeacon(Beacon):
                'dstcall',
                'receiver_name',
                'timestamp',
+               
+               # 'raw_message',
+               # 'reference_timestamp',
 
                'version',
                'platform',
@@ -86,8 +86,6 @@ class ReceiverBeacon(Beacon):
                'cpu_temp',
                'senders_visible',
                'senders_total',
-               # 'rec_crystal_correction',
-               # 'rec_crystal_correction_fine',
                'rec_input_noise',
                'senders_signal',
                'senders_messages',
@@ -103,6 +101,9 @@ class ReceiverBeacon(Beacon):
             self.dstcall,
             self.receiver_name,
             self.timestamp,
+            
+            # self.raw_message,
+            # self.reference_timestamp,
 
             self.version,
             self.platform,
@@ -116,11 +117,11 @@ class ReceiverBeacon(Beacon):
             self.cpu_temp,
             int(self.senders_visible) if self.senders_visible else None,
             int(self.senders_total) if self.senders_visible else None,
-            # self.rec_crystal_correction,
-            # self.rec_crystal_correction_fine,
             self.rec_input_noise,
             self.senders_signal,
             int(self.senders_messages) if self.senders_messages else None,
             self.good_senders_signal,
             int(self.good_senders) if self.good_senders else None,
             int(self.good_and_bad_senders) if self.good_and_bad_senders else None]
+
+Index('ix_receiver_beacons_date_receiver_id', func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id)

ogn/model/receiver_stats.py

@@ -12,10 +12,7 @@ class ReceiverStats(Base):
 
     date = Column(Date)
 
-    # Statistic data
+    # Static data
-    aircraft_beacon_count = Column(Integer)
-    aircraft_count = Column(SmallInteger)
-    max_distance = Column(Float)
     firstseen = Column(DateTime, index=True)
     lastseen = Column(DateTime, index=True)
     location_wkt = Column('location', Geometry('POINT', srid=4326))
@@ -23,6 +20,19 @@ class ReceiverStats(Base):
     version = Column(String)
     platform = Column(String)
 
+    # Statistic data
+    aircraft_beacon_count = Column(Integer)
+    aircraft_count = Column(SmallInteger)
+    max_distance = Column(Float)
+    
+    # Ranking data
+    aircraft_beacon_count_ranking_worldwide = Column(SmallInteger)
+    aircraft_beacon_count_ranking_country = Column(SmallInteger)
+    aircraft_count_ranking_worldwide = Column(SmallInteger)
+    aircraft_count_ranking_country = Column(SmallInteger)
+    max_distance_ranking_worldwide = Column(SmallInteger)
+    max_distance_ranking_country = Column(SmallInteger)
+
     # Relations
     receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
     receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='stats')

ogn/utils.py

@@ -5,7 +5,7 @@ from io import StringIO
 from aerofiles.seeyou import Reader
 from geopy.exc import GeopyError
 from geopy.geocoders import Nominatim
-from ogn.parser.utils import feet2m
+from ogn.parser.utils import FEETS_TO_METER
 import requests
 
 from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location
@@ -89,7 +89,7 @@ def get_airports(cupfile):
                     airport.location_wkt = location.to_wkt()
                     airport.altitude = waypoint['elevation']['value']
                     if (waypoint['elevation']['unit'] == 'ft'):
-                        airport.altitude = airport.altitude * feet2m
+                        airport.altitude = airport.altitude * FEETS_TO_METER
                     airport.runway_direction = waypoint['runway_direction']
                     airport.runway_length = waypoint['runway_length']['value']
                     if (waypoint['runway_length']['unit'] == 'nm'):
@@ -116,3 +116,27 @@ def open_file(filename):
     else:
         f = open(filename, 'rt')
         return f
+    
+from math import radians, cos, sin, asin, sqrt, atan2, degrees
+
+def haversine(lat1, lon1, lat2, lon2):
+    """
+    Calculate the great circle distance between two points 
+    on the earth (specified in decimal degrees)
+    """
+    # convert decimal degrees to radians 
+    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
+
+    # haversine formula 
+    dlon = lon2 - lon1 
+    dlat = lat2 - lat1 
+    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
+    c = 2 * asin(sqrt(a)) 
+    r = 6371000.785 # Radius of earth in meters
+    d = c * r
+    
+    # calculate bearing
+    bearing = atan2(sin(dlon)*cos(lat2), cos(lat1)*sin(lat2)-sin(lat1)*cos(lat2)*cos(dlon))
+    bearing = (degrees(bearing) + 360) % 360
+    
+    return d,bearing

setup.py

@@ -40,10 +40,9 @@ setup(
         'aerofiles==0.4',
         'geoalchemy2==0.4.0',
         'shapely>=1.5.17,<1.6',
-        'ogn-client==0.8.2',
+        'ogn-client==0.9.0',
         'psycopg2==2.7.3.2',
-        'mgrs==1.3.5',
+        'mgrs==1.3.5'
-        'haversine==0.4.5'
     ],
     extras_require={
         'dev': [

tests/gateway/test_merger.py

@@ -0,0 +1,101 @@
+import datetime
+import unittest
+from unittest.mock import MagicMock, call
+
+from ogn.gateway.process_tools import Merger
+
+
+class MergerTest(unittest.TestCase):
+    def test_different_keys(self):
+        a = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        b = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        c = {'name': 'John', 'receiver_name': 'Observer2', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        d = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 46)}
+
+        callback = MagicMock()
+        merger = Merger(callback=callback)
+        merger.add_message(a)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(b)
+        callback.add_message.assert_not_called()
+        
+        merger.add_message(c)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(d)
+        callback.add_message.assert_called_once_with(b)
+
+        merger.flush()
+        calls = [call(a), call(c), call(d)]
+        callback.add_message.assert_has_calls(calls, any_order=True)
+
+    def test_pair(self):
+        a = {'name': 'Jeff', 'receiver_name': 'Observer1','timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': None, 'field_b': 3.141}
+        b = {'name': 'Jeff', 'receiver_name': 'Observer1','timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': 'WTF', 'field_c': None, 'field_d': 1.4142}
+
+        merged = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': 'WTF', 'field_b': 3.141, 'field_d': 1.4142}
+
+        callback = MagicMock()
+        merger = Merger(callback=callback)
+        merger.add_message(a)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(b)
+        callback.add_message.assert_called_once_with(merged)
+
+        merger.flush()
+        callback.add_message.assert_called_once_with(merged)
+
+    def test_exceed_timedelta(self):
+        a = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        b = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        c = {'name': 'Fred', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 59)}
+
+        callback = MagicMock()
+        merger = Merger(callback=callback, max_timedelta=datetime.timedelta(seconds=10))
+        merger.add_message(a)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(b)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(c)
+        calls = [call(a), call(b)]
+        callback.add_message.assert_has_calls(calls, any_order=True)
+
+        merger.flush()
+        calls = [call(a), call(b), call(c)]
+        callback.add_message.assert_has_calls(calls, any_order=True)
+
+    def test_exceed_maxlines(self):
+        a = {'name': 'Albert', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
+        b = {'name': 'Bertram', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 46)}
+        c = {'name': 'Chlodwig', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 47)}
+        d = {'name': 'Dagobert', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 48)}
+        e = {'name': 'Erich', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 49)}
+        f = {'name': 'Frodo', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 50)}
+
+        callback = MagicMock()
+        merger = Merger(callback=callback, max_lines=5)
+        merger.add_message(a)
+        callback.add_message.assert_not_called()
+
+        merger.add_message(b)
+        callback.add_message.assert_not_called()
+        
+        merger.add_message(c)
+        callback.add_message.assert_not_called()
+        
+        merger.add_message(d)
+        callback.add_message.assert_not_called()
+        
+        merger.add_message(e)
+        callback.add_message.assert_not_called()
+        
+        merger.add_message(f)
+        callback.add_message.assert_called_once_with(a)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/gateway/test_process.py

@@ -2,7 +2,7 @@ import datetime
 import unittest
 import unittest.mock as mock
 
-from ogn.gateway.process import process_beacon, message_to_beacon
+from ogn.gateway.process import process_raw_message, string_to_message
 
 
 class ProcessManagerTest(unittest.TestCase):
@@ -16,41 +16,13 @@ class ProcessManagerTest(unittest.TestCase):
         string1 = "ICA3DD6CD>APRS,qAS,Moosburg:/195919h4820.93N/01151.39EX264/127/A=002204 !W20! id0D3DD6CD -712fpm -0.1rot 8.5dB 0e -2.1kHz gps2x2"
         string2 = "ICA3DD6CD>APRS,qAS,Moosburg:/195925h4820.90N/01151.07EX263/126/A=002139 !W74! id0D3DD6CD -712fpm +0.0rot 7.8dB 1e -2.1kHz"
 
-        process_beacon(string1)
+        process_raw_message(string1)
         mock_session.bulk_save_objects.assert_not_called()
 
         mock_datetime.utcnow.return_value = datetime.datetime(2015, 1, 1, 10, 0, 1)     # one second later
-        process_beacon(string2)
+        process_raw_message(string2)
         self.assertEqual(mock_session.bulk_save_objects.call_count, 1)
 
-    def test_message_to_beacon_brother(self):
-        string1 = "LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:/132457h4849.09NI01708.30E&/A=000528"
-        string2 = "LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:>132457h v0.2.7.arm CPU:0.9 RAM:75.3/253.6MB NTP:2.0ms/-15.2ppm +0.1C 2/2Acfts[1h] RF:+77+1.7ppm/+2.34dB/+6.5dB@10km[5411]/+10.1dB@10km[3/5]"
-        string3 = "BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:/132507h4509.60NI00919.20E&/A=000246"
-        string4 = "BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:>132507h v0.2.7.RPI-GPU CPU:1.2 RAM:35.7/455.2MB NTP:2.5ms/-5.3ppm +67.0C 1/1Acfts[1h] RF:+79+8.8ppm/+4.97dB/-0.0dB@10km[299]/+4.9dB@10km[2/32]"
-        string5 = "Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:/132624h4607.70NI00610.41E&/A=004198 Antenna: chinese, on a pylon, 20 meter above ground"
-        string6 = "Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:>132624h v0.2.7.arm CPU:1.7 RAM:812.3/1022.5MB NTP:1.8ms/+4.5ppm 0.000V 0.000A 3/4Acfts[1h] RF:+67+2.9ppm/+4.18dB/+11.7dB@10km[5018]/+17.2dB@10km[8/16]"
-
-        beacon = message_to_beacon(string1, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNone(beacon)
-
-        beacon = message_to_beacon(string2, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNotNone(beacon)
-        self.assertEqual(beacon.aprs_type, 'merged')
-
-        beacon = message_to_beacon(string3, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNone(beacon)
-
-        beacon = message_to_beacon(string4, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNotNone(beacon)
-        self.assertEqual(beacon.aprs_type, 'merged')
-
-        beacon = message_to_beacon(string5, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNone(beacon)
-
-        beacon = message_to_beacon(string6, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
-        self.assertIsNotNone(beacon)
-        self.assertEqual(beacon.aprs_type, 'merged')
 
 if __name__ == '__main__':
     unittest.main()

tests/gateway/test_saver.py

@@ -0,0 +1,42 @@
+import time
+import unittest
+from unittest.mock import MagicMock
+
+from ogn.gateway.process_tools import DbSaver, FileSaver
+
+
+class DbSaverTest(unittest.TestCase):
+    def test(self):
+        a = "Albert"
+        b = "Bertram"
+        c = "Caspar"
+        
+        session = MagicMock()
+        saver = DbSaver(session=session)
+        saver.add_message(a)
+        session.bulk_save_objects.assert_not_called()
+
+        saver.add_message(b)
+        session.bulk_save_objects.assert_not_called()
+
+        saver.add_message(c)
+        saver.flush()
+        session.bulk_save_objects.assert_called_once_with([a, b, c])
+
+    def test_timeout(self):
+        a = "Xanthippe"
+        b = "Yvonne"
+        
+        session = MagicMock()
+        saver = DbSaver(session=session)
+        saver.add_message(a)
+        session.bulk_save_objects.assert_not_called()
+
+        time.sleep(1)
+
+        saver.add_message(b)
+        session.bulk_save_objects.assert_called_once_with([a, b])
+        
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_utils.py

@@ -2,7 +2,7 @@ import os
 import unittest
 
 from ogn.model import AircraftType
-from ogn.utils import get_ddb, get_trackable, get_country_code, get_airports
+from ogn.utils import get_ddb, get_trackable, get_country_code, get_airports, haversine
 import unittest.mock as mock
 
 
@@ -57,3 +57,8 @@ class TestStringMethods(unittest.TestCase):
     def test_get_airports(self):
         airports = get_airports(os.path.dirname(__file__) + '/SeeYou.cup')
         self.assertGreater(len(airports), 1000)
+    
+    def test_haversine(self):
+        distance,bearing = haversine(45.7597, 4.8422, 48.8567, 2.3508)
+        self.assertAlmostEqual(distance, 392216.7, 0)
+        self.assertAlmostEqual(bearing, 332, 0)