Precalculate distance for bulkimport with haversine (similar to ST_Distance_Sphere)

ogn/commands/bulkimport.py

@@ -74,6 +74,8 @@ def convert(sourcefile, path=''):
     wr_rb = csv.writer(fout_rb, delimiter=',')
     wr_rb.writerow(ReceiverBeacon.get_csv_columns())
 
+    receivers = dict()
+
     print('Start importing ogn-logfile')
     for line in fin:
         num_lines += 1
@@ -89,7 +91,7 @@ def convert(sourcefile, path=''):
                     wr_rb.writerow(beacon.get_csv_values())
                 receiver_beacons = list()
 
-        beacon = message_to_beacon(line.strip(), reference_date=reference_date)
+        beacon = message_to_beacon(line.strip(), reference_date=reference_date, receivers=receivers)
         if beacon is not None:
             if isinstance(beacon, AircraftBeacon):
                 aircraft_beacons.append(beacon)
@@ -238,6 +240,7 @@ def import_aircraft_beacon_logfile(csv_logfile):
         real_address character varying(6),
         signal_power double precision,
 
+        distance double precision,
         location_mgrs character varying(15)
         );
     """
@@ -285,10 +288,10 @@ def import_aircraft_beacon_logfile(csv_logfile):
 
     session.execute("""
         INSERT INTO aircraft_beacons(location, altitude, name, receiver_name, dstcall, timestamp, track, ground_speed,
-                                    address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power,
+                                    address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power, distance, location_mgrs,
                                     receiver_id, device_id)
         SELECT t.location, t.altitude, t.name, t.receiver_name, t.dstcall, t.timestamp, t.track, t.ground_speed,
-               t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power,
+               t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power, t.distance, t.location_mgrs,
                r.id, d.id
         FROM aircraft_beacons_temp t, receivers r, devices d
         WHERE t.receiver_name = r.name AND t.address = d.address

ogn/gateway/process.py

@@ -12,10 +12,15 @@ logger = logging.getLogger(__name__)
 myMGRS = MGRS()
 
 
-def replace_lonlat_with_wkt(message):
+def replace_lonlat_with_wkt(message, reference_position=None):
+    from haversine import haversine
+
     latitude = message['latitude']
     longitude = message['longitude']
 
+    if reference_position is not None:
+        message['distance'] = 1000.0 * haversine((reference_position['latitude'], reference_position['longitude']), (latitude, longitude))
+
     location = Location(longitude, latitude)
     message['location_wkt'] = location.to_wkt()
     message['location_mgrs'] = myMGRS.toMGRS(latitude, longitude).decode('utf-8')
@@ -24,14 +29,19 @@ def replace_lonlat_with_wkt(message):
     return message
 
 
-def message_to_beacon(raw_message, reference_date):
+def message_to_beacon(raw_message, reference_date, receivers=None):
     beacon = None
 
     if raw_message[0] != '#':
         try:
             message = parse(raw_message, reference_date)
             if message['aprs_type'] == 'position':
-                message = replace_lonlat_with_wkt(message)
+                if message['beacon_type'] == 'receiver_beacon':
+                    receivers.update({message['name']: {'latitude': message['latitude'], 'longitude': message['longitude']}})
+                    message = replace_lonlat_with_wkt(message)
+                elif message['beacon_type'] == 'aircraft_beacon':
+                    reference_receiver = receivers.get(message['receiver_name'])
+                    message = replace_lonlat_with_wkt(message, reference_position=reference_receiver)
 
             if message['beacon_type'] == 'aircraft_beacon':
                 beacon = AircraftBeacon(**message)

ogn/model/aircraft_beacon.py

@@ -74,6 +74,7 @@ class AircraftBeacon(Beacon):
             self.real_address,
             self.signal_power,
 
+            self.distance,
             self.location_mgrs)
 
     @classmethod
@@ -103,6 +104,7 @@ class AircraftBeacon(Beacon):
                'real_address',
                'signal_power',
 
+               'distance',
                'location_mgrs']
 
     def get_csv_values(self):
@@ -132,4 +134,5 @@ class AircraftBeacon(Beacon):
             self.real_address,
             self.signal_power,
 
+            self.distance,
             self.location_mgrs]

setup.py

@@ -42,7 +42,8 @@ setup(
         'shapely>=1.5.17,<1.6',
         'ogn-client==0.8.2',
         'psycopg2==2.7.3.2',
-        'mgrs==1.3.5'
+        'mgrs==1.3.5',
+        'haversine==0.4.5'
     ],
     extras_require={
         'dev': [