kopia lustrzana https://github.com/glidernet/ogn-python
Several bugfixes and improvements
rodzic
945e11b615
commit
796bc45eb2
|
@ -1,10 +1,12 @@
|
|||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
from manager import Manager
|
||||
from ogn.commands.dbutils import session
|
||||
from ogn.model import AircraftBeacon, ReceiverBeacon
|
||||
from ogn.utils import open_file
|
||||
from ogn.gateway.process_tools import FileSaver, Converter, Merger
|
||||
|
||||
|
||||
manager = Manager()
|
||||
|
@ -13,118 +15,101 @@ PATTERN = '^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$'
|
|||
|
||||
|
||||
@manager.command
|
||||
def convert_logfile(path, logfile='main.log', loglevel='INFO'):
|
||||
def convert_logfile(path):
|
||||
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
|
||||
|
||||
logging.basicConfig(filename='convert.log', level=logging.DEBUG)
|
||||
|
||||
if os.path.isfile(path):
|
||||
head, tail = os.path.split(path)
|
||||
convert(tail, path=head)
|
||||
print("Finished")
|
||||
logging.info("Finished converting single file {}".format(head))
|
||||
elif os.path.isdir(path):
|
||||
for filename in os.listdir(path):
|
||||
convert(filename, path=path)
|
||||
print("Finished")
|
||||
logging.info("Finished converting file path {}".format(path))
|
||||
else:
|
||||
print("Not a file nor a path: {}".format(path))
|
||||
logging.warning("Not a file nor a path: {}".format(path))
|
||||
|
||||
|
||||
def convert(sourcefile, path=''):
|
||||
import csv
|
||||
import gzip
|
||||
logging.info("convert: {} {}".format(sourcefile, path))
|
||||
import datetime
|
||||
|
||||
from ogn.gateway.process import message_to_beacon
|
||||
from ogn.gateway.process import string_to_message
|
||||
|
||||
match = re.search(PATTERN, sourcefile)
|
||||
if match:
|
||||
reference_date_string = match.group(1)
|
||||
reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d")
|
||||
|
||||
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
|
||||
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
|
||||
# Build the processing pipeline
|
||||
saver = FileSaver()
|
||||
converter = Converter(callback=saver)
|
||||
merger = Merger(callback=converter)
|
||||
|
||||
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
|
||||
print("Reading file: {}".format(sourcefile))
|
||||
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
|
||||
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
|
||||
else:
|
||||
print("Output files for file {} already exists. Skipping".format(sourcefile))
|
||||
try:
|
||||
saver.open(path, reference_date_string)
|
||||
except FileExistsError:
|
||||
logging.warning("Output files already exists. Skipping")
|
||||
return
|
||||
else:
|
||||
print("filename '{}' does not match pattern. Skipping".format(sourcefile))
|
||||
logging.warning("filename '{}' does not match pattern. Skipping".format(sourcefile))
|
||||
return
|
||||
|
||||
fin = open_file(os.path.join(path, sourcefile))
|
||||
|
||||
# get total lines of the input file
|
||||
total = 0
|
||||
total_lines = 0
|
||||
for line in fin:
|
||||
total += 1
|
||||
total_lines += 1
|
||||
fin.seek(0)
|
||||
|
||||
aircraft_beacons = list()
|
||||
receiver_beacons = list()
|
||||
|
||||
progress = -1
|
||||
num_lines = 0
|
||||
|
||||
wr_ab = csv.writer(fout_ab, delimiter=',')
|
||||
wr_ab.writerow(AircraftBeacon.get_csv_columns())
|
||||
|
||||
wr_rb = csv.writer(fout_rb, delimiter=',')
|
||||
wr_rb.writerow(ReceiverBeacon.get_csv_columns())
|
||||
current_line = 0
|
||||
|
||||
print('Start importing ogn-logfile')
|
||||
for line in fin:
|
||||
num_lines += 1
|
||||
if int(100 * num_lines / total) != progress:
|
||||
progress = round(100 * num_lines / total)
|
||||
print("\rReading line {} ({}%)".format(num_lines, progress), end='')
|
||||
if len(aircraft_beacons) > 0:
|
||||
for beacon in aircraft_beacons:
|
||||
wr_ab.writerow(beacon.get_csv_values())
|
||||
aircraft_beacons = list()
|
||||
if len(receiver_beacons) > 0:
|
||||
for beacon in receiver_beacons:
|
||||
wr_rb.writerow(beacon.get_csv_values())
|
||||
receiver_beacons = list()
|
||||
current_line += 1
|
||||
if int(1000 * current_line / total_lines) != progress:
|
||||
progress = round(1000 * current_line / total_lines)
|
||||
print("\rReading line {} ({}%)".format(current_line, progress / 10), end='')
|
||||
|
||||
beacon = message_to_beacon(line.strip(), reference_date=reference_date, wait_for_brother=True)
|
||||
if beacon is not None:
|
||||
if isinstance(beacon, AircraftBeacon):
|
||||
aircraft_beacons.append(beacon)
|
||||
elif isinstance(beacon, ReceiverBeacon):
|
||||
receiver_beacons.append(beacon)
|
||||
message = string_to_message(line.strip(), reference_date=reference_date)
|
||||
if message is None:
|
||||
print("=====")
|
||||
print(line.strip())
|
||||
continue
|
||||
|
||||
merger.add_message(message)
|
||||
|
||||
if len(aircraft_beacons) > 0:
|
||||
for beacon in aircraft_beacons:
|
||||
wr_ab.writerow(beacon.get_csv_values())
|
||||
if len(receiver_beacons) > 0:
|
||||
for beacon in receiver_beacons:
|
||||
wr_rb.writerow(beacon.get_csv_values())
|
||||
merger.flush()
|
||||
saver.close()
|
||||
|
||||
fin.close()
|
||||
fout_ab.close()
|
||||
fout_rb.close()
|
||||
|
||||
|
||||
@manager.command
|
||||
def drop_indices():
|
||||
"""Drop indices of AircraftBeacon."""
|
||||
session.execute("""
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_location;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs;
|
||||
DROP INDEX IF EXISTS idx_aircraft_beacons_location;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_date_device_id_address;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_date_receiver_id_distance;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp;
|
||||
|
||||
DROP INDEX IF EXISTS idx_receiver_beacons_location;
|
||||
DROP INDEX IF EXISTS ix_receiver_beacons_date_receiver_id;
|
||||
DROP INDEX IF EXISTS ix_receiver_beacons_timestamp;
|
||||
""")
|
||||
print("Dropped indices of AircraftBeacon")
|
||||
print("Dropped indices of AircraftBeacon and ReceiverBeacon")
|
||||
|
||||
# disable constraint trigger
|
||||
session.execute("""
|
||||
ALTER TABLE aircraft_beacons DISABLE TRIGGER ALL
|
||||
ALTER TABLE aircraft_beacons DISABLE TRIGGER ALL;
|
||||
ALTER TABLE receiver_beacons DISABLE TRIGGER ALL;
|
||||
""")
|
||||
session.commit()
|
||||
print("Disabled constraint triggers")
|
||||
|
||||
|
||||
|
@ -132,18 +117,22 @@ def drop_indices():
|
|||
def create_indices():
|
||||
"""Create indices for AircraftBeacon."""
|
||||
session.execute("""
|
||||
CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name);
|
||||
CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address);
|
||||
CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp);
|
||||
CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
|
||||
|
||||
CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance)
|
||||
CREATE INDEX idx_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
|
||||
CREATE INDEX ix_aircraft_beacons_date_device_id_address ON aircraft_beacons USING BTREE((timestamp::date), device_id, address);
|
||||
CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING BTREE((timestamp::date), receiver_id, distance);
|
||||
CREATE INDEX ix_aircraft_beacons_timestamp ON aircraft_beacons USING BTREE(timestamp);
|
||||
|
||||
CREATE INDEX idx_receiver_beacons_location ON receiver_beacons USING GIST(location);
|
||||
CREATE INDEX ix_receiver_beacons_date_receiver_id ON receiver_beacons USING BTREE((timestamp::date), receiver_id);
|
||||
CREATE INDEX ix_receiver_beacons_timestamp ON receiver_beacons USING BTREE(timestamp);
|
||||
""")
|
||||
print("Created indices for AircraftBeacon")
|
||||
print("Created indices for AircraftBeacon and ReceiverBeacon")
|
||||
|
||||
session.execute("""
|
||||
ALTER TABLE aircraft_beacons ENABLE TRIGGER ALL
|
||||
ALTER TABLE aircraft_beacons ENABLE TRIGGER ALL;
|
||||
ALTER TABLE receiver_beacons ENABLE TRIGGER ALL;
|
||||
""")
|
||||
session.commit()
|
||||
print("Enabled constraint triggers")
|
||||
|
||||
|
||||
|
@ -198,6 +187,11 @@ def import_logfile(path):
|
|||
else:
|
||||
print("For {} beacons already exist. Skipping".format(reference_date_string))
|
||||
else:
|
||||
s1 = header
|
||||
s2 = ','.join(AircraftBeacon.get_csv_columns())
|
||||
print(s1)
|
||||
print(s2)
|
||||
print([i for i in range(len(s1)) if s1[i] != s2[i]])
|
||||
print("Unknown file type: {}".format(tail))
|
||||
|
||||
|
||||
|
@ -214,32 +208,34 @@ def import_aircraft_beacon_logfile(csv_logfile):
|
|||
DROP TABLE IF EXISTS aircraft_beacons_temp;
|
||||
CREATE TABLE aircraft_beacons_temp(
|
||||
location geometry,
|
||||
altitude integer,
|
||||
altitude real,
|
||||
name character varying,
|
||||
dstcall character varying,
|
||||
relay character varying,
|
||||
receiver_name character varying(9),
|
||||
"timestamp" timestamp without time zone,
|
||||
track integer,
|
||||
ground_speed double precision,
|
||||
track smallint,
|
||||
ground_speed real,
|
||||
|
||||
address_type smallint,
|
||||
aircraft_type smallint,
|
||||
stealth boolean,
|
||||
address character varying(6),
|
||||
climb_rate double precision,
|
||||
turn_rate double precision,
|
||||
flightlevel double precision,
|
||||
signal_quality double precision,
|
||||
error_count integer,
|
||||
frequency_offset double precision,
|
||||
gps_status character varying,
|
||||
software_version double precision,
|
||||
address character varying,
|
||||
climb_rate real,
|
||||
turn_rate real,
|
||||
signal_quality real,
|
||||
error_count smallint,
|
||||
frequency_offset real,
|
||||
gps_quality_horizontal smallint,
|
||||
gps_quality_vertical smallint,
|
||||
software_version real,
|
||||
hardware_version smallint,
|
||||
real_address character varying(6),
|
||||
signal_power double precision,
|
||||
signal_power real,
|
||||
|
||||
distance double precision,
|
||||
distance real,
|
||||
radial smallint,
|
||||
normalized_signal_quality real,
|
||||
location_mgrs character varying(15)
|
||||
);
|
||||
"""
|
||||
|
@ -287,10 +283,12 @@ def import_aircraft_beacon_logfile(csv_logfile):
|
|||
|
||||
session.execute("""
|
||||
INSERT INTO aircraft_beacons(location, altitude, name, dstcall, relay, receiver_name, timestamp, track, ground_speed,
|
||||
address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power, distance, location_mgrs,
|
||||
address_type, aircraft_type, stealth, address, climb_rate, turn_rate, signal_quality, error_count, frequency_offset, gps_quality_horizontal, gps_quality_vertical, software_version, hardware_version, real_address, signal_power,
|
||||
distance, radial, normalized_signal_quality, location_mgrs,
|
||||
receiver_id, device_id)
|
||||
SELECT t.location, t.altitude, t.name, t.dstcall, t.relay, t.receiver_name, t.timestamp, t.track, t.ground_speed,
|
||||
t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power, t.distance, t.location_mgrs,
|
||||
t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.signal_quality, t.error_count, t.frequency_offset, t.gps_quality_horizontal, t.gps_quality_vertical, t.software_version, t.hardware_version, t.real_address, t.signal_power,
|
||||
t.distance, t.radial, t.normalized_signal_quality, t.location_mgrs,
|
||||
r.id, d.id
|
||||
FROM aircraft_beacons_temp t, receivers r, devices d
|
||||
WHERE t.receiver_name = r.name AND t.address = d.address
|
||||
|
@ -311,7 +309,7 @@ def import_receiver_beacon_logfile(csv_logfile):
|
|||
DROP TABLE IF EXISTS receiver_beacons_temp;
|
||||
CREATE TABLE receiver_beacons_temp(
|
||||
location geometry,
|
||||
altitude integer,
|
||||
altitude real,
|
||||
name character varying,
|
||||
receiver_name character varying(9),
|
||||
dstcall character varying,
|
||||
|
@ -319,20 +317,20 @@ def import_receiver_beacon_logfile(csv_logfile):
|
|||
|
||||
version character varying,
|
||||
platform character varying,
|
||||
cpu_load double precision,
|
||||
free_ram double precision,
|
||||
total_ram double precision,
|
||||
ntp_error double precision,
|
||||
rt_crystal_correction double precision,
|
||||
voltage double precision,
|
||||
amperage double precision,
|
||||
cpu_temp double precision,
|
||||
cpu_load real,
|
||||
free_ram real,
|
||||
total_ram real,
|
||||
ntp_error real,
|
||||
rt_crystal_correction real,
|
||||
voltage real,
|
||||
amperage real,
|
||||
cpu_temp real,
|
||||
senders_visible integer,
|
||||
senders_total integer,
|
||||
rec_input_noise double precision,
|
||||
senders_signal double precision,
|
||||
rec_input_noise real,
|
||||
senders_signal real,
|
||||
senders_messages integer,
|
||||
good_senders_signal double precision,
|
||||
good_senders_signal real,
|
||||
good_senders integer,
|
||||
good_and_bad_senders integer
|
||||
);
|
||||
|
|
|
@ -22,8 +22,8 @@ def init():
|
|||
session.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
|
||||
session.commit()
|
||||
Base.metadata.create_all(engine)
|
||||
alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
|
||||
command.stamp(alembic_cfg, "head")
|
||||
#alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
|
||||
#command.stamp(alembic_cfg, "head")
|
||||
print("Done.")
|
||||
|
||||
|
||||
|
|
|
@ -1,24 +1,30 @@
|
|||
import logging
|
||||
from math import log10
|
||||
|
||||
from mgrs import MGRS
|
||||
from haversine import haversine
|
||||
|
||||
from ogn.utils import haversine
|
||||
from ogn.commands.dbutils import session
|
||||
from ogn.model import AircraftBeacon, ReceiverBeacon, Location
|
||||
from ogn.parser import parse, ParseError
|
||||
from datetime import datetime, timedelta
|
||||
from ogn.gateway.process_tools import DbSaver, Converter, DummyMerger, AIRCRAFT_TYPES, RECEIVER_TYPES
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
myMGRS = MGRS()
|
||||
|
||||
|
||||
def replace_lonlat_with_wkt(message, reference_receiver=None):
|
||||
|
||||
def _replace_lonlat_with_wkt(message, reference_receiver=None):
|
||||
latitude = message['latitude']
|
||||
longitude = message['longitude']
|
||||
|
||||
if reference_receiver is not None:
|
||||
message['distance'] = 1000.0 * haversine((reference_receiver['latitude'], reference_receiver['longitude']), (latitude, longitude))
|
||||
distance,bearing = haversine(reference_receiver['latitude'], reference_receiver['longitude'], latitude, longitude)
|
||||
message['distance'] = distance
|
||||
message['radial'] = round(bearing)
|
||||
if message['signal_quality'] is not None and distance >= 1:
|
||||
message['normalized_signal_quality'] = message['signal_quality'] + 20 * log10(message['distance'] / 10000) # normalized to 10km
|
||||
|
||||
location = Location(longitude, latitude)
|
||||
message['location_wkt'] = location.to_wkt()
|
||||
|
@ -27,88 +33,55 @@ def replace_lonlat_with_wkt(message, reference_receiver=None):
|
|||
del message['longitude']
|
||||
return message
|
||||
|
||||
previous_message = None
|
||||
|
||||
receivers = dict()
|
||||
|
||||
|
||||
def message_to_beacon(raw_message, reference_date, wait_for_brother=False):
|
||||
beacon = None
|
||||
global previous_message
|
||||
def string_to_message(raw_string, reference_date):
|
||||
global receivers
|
||||
|
||||
if raw_message[0] != '#':
|
||||
try:
|
||||
message = parse(raw_message, reference_date)
|
||||
except NotImplementedError as e:
|
||||
logger.error('Received message: {}'.format(raw_message))
|
||||
logger.error(e)
|
||||
return None
|
||||
except ParseError as e:
|
||||
logger.error('Received message: {}'.format(raw_message))
|
||||
logger.error('Drop packet, {}'.format(e.message))
|
||||
return None
|
||||
except TypeError as e:
|
||||
logger.error('TypeError: {}'.format(raw_message))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(raw_message)
|
||||
logger.error(e)
|
||||
return None
|
||||
try:
|
||||
message = parse(raw_string, reference_date)
|
||||
except NotImplementedError as e:
|
||||
logger.error('No parser implemented for message: {}'.format(raw_string))
|
||||
return None
|
||||
except ParseError as e:
|
||||
logger.error('Parsing error with message: {}'.format(raw_string))
|
||||
return None
|
||||
except TypeError as e:
|
||||
logger.error('TypeError with message: {}'.format(raw_string))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(raw_string)
|
||||
logger.error(e)
|
||||
return None
|
||||
|
||||
# update reference receivers and distance to the receiver
|
||||
if message['aprs_type'] == 'position':
|
||||
if message['beacon_type'] in ['receiver_beacon', 'aprs_receiver', 'receiver']:
|
||||
receivers.update({message['name']: {'latitude': message['latitude'], 'longitude': message['longitude']}})
|
||||
message = replace_lonlat_with_wkt(message)
|
||||
elif message['beacon_type'] in ['aircraft_beacon', 'aprs_aircraft', 'flarm', 'tracker']:
|
||||
reference_receiver = receivers.get(message['receiver_name'])
|
||||
message = replace_lonlat_with_wkt(message, reference_receiver=reference_receiver)
|
||||
# update reference receivers and distance to the receiver
|
||||
if message['aprs_type'] == 'position':
|
||||
if message['beacon_type'] in RECEIVER_TYPES:
|
||||
receivers.update({message['name']: {'latitude': message['latitude'], 'longitude': message['longitude']}})
|
||||
message = _replace_lonlat_with_wkt(message)
|
||||
elif message['beacon_type'] in AIRCRAFT_TYPES:
|
||||
reference_receiver = receivers.get(message['receiver_name'])
|
||||
message = _replace_lonlat_with_wkt(message, reference_receiver=reference_receiver)
|
||||
if 'gps_quality' in message and message['gps_quality'] is not None and 'horizontal' in message['gps_quality']:
|
||||
message['gps_quality_horizontal'] = message['gps_quality']['horizontal']
|
||||
message['gps_quality_vertical'] = message['gps_quality']['vertical']
|
||||
|
||||
# optional: merge different beacon types
|
||||
params = dict()
|
||||
if wait_for_brother is True:
|
||||
if previous_message is None:
|
||||
previous_message = message
|
||||
return None
|
||||
elif message['name'] == previous_message['name'] and message['timestamp'] == previous_message['timestamp']:
|
||||
params = message
|
||||
params.update(previous_message)
|
||||
params['aprs_type'] = 'merged'
|
||||
previous_message = None
|
||||
else:
|
||||
params = previous_message
|
||||
previous_message = message
|
||||
else:
|
||||
params = message
|
||||
# update raw_message
|
||||
message['raw_message'] = raw_string
|
||||
|
||||
# create beacons
|
||||
if params['beacon_type'] in ['aircraft_beacon', 'aprs_aircraft', 'flarm', 'tracker']:
|
||||
beacon = AircraftBeacon(**params)
|
||||
elif params['beacon_type'] in ['receiver_beacon', 'aprs_receiver', 'receiver']:
|
||||
beacon = ReceiverBeacon(**params)
|
||||
else:
|
||||
print("Whoops: what is this: {}".format(params))
|
||||
|
||||
return beacon
|
||||
|
||||
beacons = list()
|
||||
last_commit = datetime.utcnow()
|
||||
return message
|
||||
|
||||
|
||||
def process_beacon(raw_message, reference_date=None):
|
||||
global beacons
|
||||
global last_commit
|
||||
# Build the processing pipeline
|
||||
saver = DbSaver(session=session)
|
||||
converter = Converter(callback=saver)
|
||||
merger = DummyMerger(callback=converter)
|
||||
|
||||
|
||||
def process_raw_message(raw_message, reference_date=None, merger=merger):
|
||||
logger.debug('Received message: {}'.format(raw_message))
|
||||
message = string_to_message(raw_message, reference_date)
|
||||
merger.add_message(message)
|
||||
|
||||
beacon = message_to_beacon(raw_message, reference_date)
|
||||
if beacon is not None:
|
||||
beacons.append(beacon)
|
||||
logger.debug('Received message: {}'.format(raw_message))
|
||||
|
||||
current_time = datetime.utcnow()
|
||||
elapsed_time = current_time - last_commit
|
||||
if elapsed_time >= timedelta(seconds=1):
|
||||
session.bulk_save_objects(beacons)
|
||||
session.commit()
|
||||
logger.debug('Commited beacons')
|
||||
beacons = list()
|
||||
last_commit = current_time
|
||||
|
|
|
@ -0,0 +1,184 @@
|
|||
from datetime import datetime, timedelta
|
||||
from ogn.model import AircraftBeacon, ReceiverBeacon
|
||||
|
||||
AIRCRAFT_TYPES = ['aprs_aircraft', 'flarm', 'tracker', 'fanet', 'lt24', 'naviter', 'skylines', 'spider', 'spot']
|
||||
RECEIVER_TYPES = ['aprs_receiver', 'receiver']
|
||||
|
||||
class DummyMerger:
|
||||
def __init__(self, callback):
|
||||
self.callback = callback
|
||||
|
||||
def add_message(self, message):
|
||||
self.callback.add_message(message)
|
||||
|
||||
def flush(self):
|
||||
pass
|
||||
|
||||
class Merger:
|
||||
def __init__(self, callback, max_timedelta=None, max_lines=None):
|
||||
self.callback = callback
|
||||
self.max_timedelta = max_timedelta
|
||||
self.max_lines = max_lines
|
||||
self.message_map = dict()
|
||||
|
||||
def add_message(self, message):
|
||||
if message is None or ('raw_message' in message and message['raw_message'][0] == '#'):
|
||||
return
|
||||
|
||||
# release old messages
|
||||
if self.max_timedelta is not None:
|
||||
for receiver,v1 in self.message_map.items():
|
||||
for name,v2 in v1.items():
|
||||
for timestamp,message in v2.items():
|
||||
if message['timestamp'] - timestamp > self.max_timedelta:
|
||||
self.callback.add_message(message)
|
||||
del self.message_map[receiver][name][timestamp]
|
||||
|
||||
# release messages > max_lines
|
||||
if self.max_lines is not None:
|
||||
pass
|
||||
|
||||
# merge messages with same timestamp
|
||||
if message['receiver_name'] in self.message_map:
|
||||
if message['name'] in self.message_map[message['receiver_name']]:
|
||||
messages = self.message_map[message['receiver_name']][message['name']]
|
||||
if message['timestamp'] in messages:
|
||||
other = messages[message['timestamp']]
|
||||
params1 = dict( [(k,v) for k,v in message.items() if v is not None])
|
||||
params2 = dict( [(k,v) for k,v in other.items() if v is not None])
|
||||
merged = {**params1, **params2}
|
||||
|
||||
# zum debuggen
|
||||
if 'raw_message' in message and 'raw_message' in other:
|
||||
merged['raw_message'] = '"{}","{}"'.format(message['raw_message'], other['raw_message'])
|
||||
|
||||
self.callback.add_message(merged)
|
||||
del self.message_map[message['receiver_name']][message['name']][message['timestamp']]
|
||||
else:
|
||||
self.message_map[message['receiver_name']][message['name']][message['timestamp']] = message
|
||||
|
||||
# release previous messages
|
||||
for timestamp in list(messages):
|
||||
if timestamp < message['timestamp']:
|
||||
self.callback.add_message(messages[timestamp])
|
||||
del self.message_map[message['receiver_name']][message['name']][timestamp]
|
||||
else:
|
||||
# add new message
|
||||
self.message_map[message['receiver_name']].update({message['name']: {message['timestamp']: message}})
|
||||
else:
|
||||
self.message_map.update({message['receiver_name']: {message['name']: {message['timestamp']: message}}})
|
||||
|
||||
def flush(self):
|
||||
for receiver,v1 in self.message_map.items():
|
||||
for name,v2 in v1.items():
|
||||
for timestamp in v2:
|
||||
self.callback.add_message(self.message_map[receiver][name][timestamp])
|
||||
|
||||
self.callback.flush()
|
||||
self.message_map = dict()
|
||||
|
||||
class Converter:
|
||||
def __init__(self, callback):
|
||||
self.callback = callback
|
||||
|
||||
def add_message(self, message):
|
||||
try:
|
||||
beacon = self.message_to_beacon(message)
|
||||
self.callback.add_message(beacon)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(message)
|
||||
return
|
||||
|
||||
def message_to_beacon(self, message):
|
||||
# create beacons
|
||||
if message['beacon_type'] in AIRCRAFT_TYPES:
|
||||
beacon = AircraftBeacon(**message)
|
||||
elif message['beacon_type'] in RECEIVER_TYPES:
|
||||
if 'rec_crystal_correction' in message:
|
||||
del message['rec_crystal_correction']
|
||||
del message['rec_crystal_correction_fine']
|
||||
beacon = ReceiverBeacon(**message)
|
||||
else:
|
||||
print("Whoops: what is this: {}".format(message))
|
||||
|
||||
return beacon
|
||||
|
||||
def flush(self):
|
||||
self.callback.flush()
|
||||
|
||||
class DummySaver:
|
||||
def add_message(self, message):
|
||||
if message['beacon_type'] != "aprs_aircraft":
|
||||
print(message['beacon_type'])
|
||||
|
||||
def flush(self):
|
||||
print("========== flush ==========")
|
||||
|
||||
class DbSaver:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
self.beacons = list()
|
||||
self.last_commit = datetime.utcnow()
|
||||
|
||||
def add_message(self, beacon):
|
||||
global last_commit
|
||||
global beacons
|
||||
|
||||
self.beacons.append(beacon)
|
||||
|
||||
elapsed_time = datetime.utcnow() - self.last_commit
|
||||
if elapsed_time >= timedelta(seconds=1):
|
||||
self.flush()
|
||||
|
||||
def flush(self):
|
||||
try:
|
||||
self.session.bulk_save_objects(self.beacons)
|
||||
self.session.commit()
|
||||
self.beacons = list()
|
||||
self.last_commit = datetime.utcnow()
|
||||
except Exception as e:
|
||||
self.session.rollback()
|
||||
print(e)
|
||||
return
|
||||
|
||||
import os, gzip, csv
|
||||
|
||||
class FileSaver:
|
||||
def __init__(self):
|
||||
self.aircraft_messages = list()
|
||||
self.receiver_messages = list()
|
||||
|
||||
def open(self, path, reference_date_string):
|
||||
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
|
||||
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
|
||||
|
||||
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
|
||||
self.fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
|
||||
self.fout_rb = gzip.open(receiver_beacon_filename, 'wt')
|
||||
else:
|
||||
raise FileExistsError
|
||||
|
||||
self.aircraft_writer = csv.writer(self.fout_ab, delimiter=',')
|
||||
self.aircraft_writer.writerow(AircraftBeacon.get_csv_columns())
|
||||
|
||||
self.receiver_writer = csv.writer(self.fout_rb, delimiter=',')
|
||||
self.receiver_writer.writerow(ReceiverBeacon.get_csv_columns())
|
||||
|
||||
return 1
|
||||
|
||||
def add_message(self, beacon):
|
||||
if isinstance(beacon, AircraftBeacon):
|
||||
self.aircraft_messages.append(beacon.get_csv_values())
|
||||
elif isinstance(beacon, ReceiverBeacon):
|
||||
self.receiver_messages.append(beacon.get_csv_values())
|
||||
|
||||
def flush(self):
|
||||
self.aircraft_writer.writerows(self.aircraft_messages)
|
||||
self.receiver_writer.writerows(self.receiver_messages)
|
||||
self.aircraft_messages = list()
|
||||
self.receiver_messages = list()
|
||||
|
||||
def close(self):
|
||||
self.fout_ab.close()
|
||||
self.fout_rb.close()
|
|
@ -7,28 +7,28 @@ from .beacon import Beacon
|
|||
class AircraftBeacon(Beacon):
|
||||
__tablename__ = "aircraft_beacons"
|
||||
|
||||
# Activate relay for AircraftBeacon
|
||||
relay = Column(String)
|
||||
|
||||
# Flarm specific data
|
||||
address_type = Column(SmallInteger)
|
||||
aircraft_type = Column(SmallInteger)
|
||||
stealth = Column(Boolean)
|
||||
address = Column(String(6))
|
||||
address = Column(String)
|
||||
climb_rate = Column(Float(precision=2))
|
||||
turn_rate = Column(Float(precision=2))
|
||||
flightlevel = Column(Float(precision=2))
|
||||
signal_quality = Column(Float(precision=2))
|
||||
error_count = Column(SmallInteger)
|
||||
frequency_offset = Column(Float(precision=2))
|
||||
gps_status = Column(String)
|
||||
gps_quality_horizontal = Column(SmallInteger)
|
||||
gps_quality_vertical = Column(SmallInteger)
|
||||
software_version = Column(Float(precision=2))
|
||||
hardware_version = Column(SmallInteger)
|
||||
real_address = Column(String(6))
|
||||
signal_power = Column(Float(precision=2))
|
||||
|
||||
# Not so very important data
|
||||
proximity = None
|
||||
|
||||
# Tracker stuff (position message)
|
||||
flightlevel = None
|
||||
|
||||
# Tracker stuff (status message)
|
||||
gps_satellites = None
|
||||
gps_quality = None
|
||||
gps_altitude = None
|
||||
|
@ -40,10 +40,16 @@ class AircraftBeacon(Beacon):
|
|||
noise_level = None
|
||||
relays = None
|
||||
|
||||
# Spider stuff
|
||||
spider_id = None
|
||||
model = None
|
||||
status = None
|
||||
|
||||
# Calculated values
|
||||
status = Column(SmallInteger, default=0)
|
||||
distance = Column(Float)
|
||||
location_mgrs = Column(String(15), index=True)
|
||||
distance = Column(Float(precision=2))
|
||||
radial = Column(SmallInteger)
|
||||
normalized_signal_quality = Column(Float(precision=2))
|
||||
location_mgrs = Column(String(15))
|
||||
|
||||
# Relations
|
||||
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'))
|
||||
|
@ -58,24 +64,26 @@ class AircraftBeacon(Beacon):
|
|||
Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
|
||||
|
||||
def __repr__(self):
|
||||
return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
self.address_type,
|
||||
self.aircraft_type,
|
||||
self.stealth,
|
||||
self.address,
|
||||
self.climb_rate,
|
||||
self.turn_rate,
|
||||
self.flightlevel,
|
||||
self.signal_quality,
|
||||
self.error_count,
|
||||
self.frequency_offset,
|
||||
self.gps_status,
|
||||
self.gps_quality_horizontal,
|
||||
self.gps_quality_vertical,
|
||||
self.software_version,
|
||||
self.hardware_version,
|
||||
self.real_address,
|
||||
self.signal_power,
|
||||
|
||||
self.distance,
|
||||
self.radial,
|
||||
self.normalized_signal_quality,
|
||||
self.location_mgrs)
|
||||
|
||||
@classmethod
|
||||
|
@ -89,6 +97,9 @@ class AircraftBeacon(Beacon):
|
|||
'timestamp',
|
||||
'track',
|
||||
'ground_speed',
|
||||
|
||||
#'raw_message',
|
||||
#'reference_timestamp',
|
||||
|
||||
'address_type',
|
||||
'aircraft_type',
|
||||
|
@ -96,23 +107,25 @@ class AircraftBeacon(Beacon):
|
|||
'address',
|
||||
'climb_rate',
|
||||
'turn_rate',
|
||||
'flightlevel',
|
||||
'signal_quality',
|
||||
'error_count',
|
||||
'frequency_offset',
|
||||
'gps_status',
|
||||
'gps_quality_horizontal',
|
||||
'gps_quality_vertical',
|
||||
'software_version',
|
||||
'hardware_version',
|
||||
'real_address',
|
||||
'signal_power',
|
||||
|
||||
'distance',
|
||||
'radial',
|
||||
'normalized_signal_quality',
|
||||
'location_mgrs']
|
||||
|
||||
def get_csv_values(self):
|
||||
return [
|
||||
self.location_wkt,
|
||||
int(self.altitude),
|
||||
int(self.altitude) if self.altitude else None,
|
||||
self.name,
|
||||
self.dstcall,
|
||||
self.relay,
|
||||
|
@ -120,6 +133,9 @@ class AircraftBeacon(Beacon):
|
|||
self.timestamp,
|
||||
self.track,
|
||||
self.ground_speed,
|
||||
|
||||
#self.raw_message,
|
||||
#self.reference_timestamp,
|
||||
|
||||
self.address_type,
|
||||
self.aircraft_type,
|
||||
|
@ -127,18 +143,21 @@ class AircraftBeacon(Beacon):
|
|||
self.address,
|
||||
self.climb_rate,
|
||||
self.turn_rate,
|
||||
self.flightlevel,
|
||||
self.signal_quality,
|
||||
self.error_count,
|
||||
self.frequency_offset,
|
||||
self.gps_status,
|
||||
self.gps_quality_horizontal,
|
||||
self.gps_quality_vertical,
|
||||
self.software_version,
|
||||
self.hardware_version,
|
||||
self.real_address,
|
||||
self.signal_power,
|
||||
|
||||
self.distance,
|
||||
self.radial,
|
||||
self.normalized_signal_quality,
|
||||
self.location_mgrs]
|
||||
|
||||
|
||||
Index('ix_aircraft_beacons_date_device_id_address', func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address)
|
||||
Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance)
|
|
@ -16,7 +16,7 @@ class Beacon(AbstractConcreteBase, Base):
|
|||
|
||||
name = Column(String)
|
||||
dstcall = Column(String)
|
||||
relay = None
|
||||
relay = Column(String)
|
||||
receiver_name = Column(String(9))
|
||||
timestamp = Column(DateTime, index=True)
|
||||
symboltable = None
|
||||
|
@ -25,10 +25,13 @@ class Beacon(AbstractConcreteBase, Base):
|
|||
ground_speed = Column(Float(precision=2))
|
||||
comment = None
|
||||
|
||||
# Type information
|
||||
beacon_type = None
|
||||
aprs_type = None
|
||||
|
||||
location_mgrs = None
|
||||
# Debug information
|
||||
raw_message = None #Column(String)
|
||||
reference_timestamp = None #Column(DateTime, index=True)
|
||||
|
||||
@property
|
||||
def location(self):
|
||||
|
|
|
@ -9,7 +9,8 @@ class Device(Base):
|
|||
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
address = Column(String(6), index=True)
|
||||
#address = Column(String(6), index=True)
|
||||
address = Column(String, index=True)
|
||||
firstseen = Column(DateTime, index=True)
|
||||
lastseen = Column(DateTime, index=True)
|
||||
aircraft_type = Column(SmallInteger, index=True)
|
||||
|
|
|
@ -9,7 +9,8 @@ class DeviceInfo(Base):
|
|||
|
||||
id = Column(Integer, primary_key=True)
|
||||
address_type = None
|
||||
address = Column(String(6), index=True)
|
||||
#address = Column(String(6), index=True)
|
||||
address = Column(String, index=True)
|
||||
aircraft = Column(String)
|
||||
registration = Column(String(7))
|
||||
competition = Column(String(3))
|
||||
|
|
|
@ -11,10 +11,7 @@ class DeviceStats(Base):
|
|||
|
||||
date = Column(Date)
|
||||
|
||||
# Statistic data
|
||||
max_altitude = Column(Float(precision=2))
|
||||
receiver_count = Column(SmallInteger)
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
# Static data
|
||||
firstseen = Column(DateTime)
|
||||
lastseen = Column(DateTime)
|
||||
aircraft_type = Column(SmallInteger)
|
||||
|
@ -23,7 +20,20 @@ class DeviceStats(Base):
|
|||
hardware_version = Column(SmallInteger)
|
||||
real_address = Column(String(6))
|
||||
|
||||
# Statistic data
|
||||
max_altitude = Column(Float(precision=2))
|
||||
receiver_count = Column(SmallInteger)
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
|
||||
ambiguous = Column(Boolean)
|
||||
|
||||
# Ranking data
|
||||
max_altitude_ranking_worldwide = Column(Integer)
|
||||
max_altitude_ranking_country = Column(Integer)
|
||||
receiver_count_ranking_worldwide = Column(Integer)
|
||||
receiver_count_ranking_country = Column(Integer)
|
||||
aircraft_beacon_count_ranking_worldwide = Column(Integer)
|
||||
aircraft_beacon_count_ranking_country = Column(Integer)
|
||||
|
||||
# Relations
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
from sqlalchemy import Column, Float, String, Integer, SmallInteger, ForeignKey, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from sqlalchemy.sql import func
|
||||
from .beacon import Beacon
|
||||
|
||||
|
||||
class ReceiverBeacon(Beacon):
|
||||
__tablename__ = "receiver_beacons"
|
||||
|
||||
# disable not so important aprs fields
|
||||
# disable irrelevant aprs fields
|
||||
track = None
|
||||
ground_speed = None
|
||||
|
||||
|
@ -24,8 +24,6 @@ class ReceiverBeacon(Beacon):
|
|||
cpu_temp = Column(Float(precision=2))
|
||||
senders_visible = Column(Integer)
|
||||
senders_total = Column(Integer)
|
||||
rec_crystal_correction = 0 # obsolete since 0.2.0
|
||||
rec_crystal_correction_fine = 0 # obsolete since 0.2.0
|
||||
rec_input_noise = Column(Float(precision=2))
|
||||
senders_signal = Column(Float(precision=2))
|
||||
senders_messages = Column(Integer)
|
||||
|
@ -33,6 +31,7 @@ class ReceiverBeacon(Beacon):
|
|||
good_senders = Column(Integer)
|
||||
good_and_bad_senders = Column(Integer)
|
||||
|
||||
# User comment: used for additional information like hardware configuration, web site, email address, ...
|
||||
user_comment = None
|
||||
|
||||
# Relations
|
||||
|
@ -43,7 +42,7 @@ class ReceiverBeacon(Beacon):
|
|||
Index('ix_receiver_beacons_receiver_id_name', 'receiver_id', 'name')
|
||||
|
||||
def __repr__(self):
|
||||
return "<ReceiverBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
return "<ReceiverBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
self.version,
|
||||
self.platform,
|
||||
self.cpu_load,
|
||||
|
@ -56,8 +55,6 @@ class ReceiverBeacon(Beacon):
|
|||
self.cpu_temp,
|
||||
self.senders_visible,
|
||||
self.senders_total,
|
||||
# self.rec_crystal_correction,
|
||||
# self.rec_crystal_correction_fine,
|
||||
self.rec_input_noise,
|
||||
self.senders_signal,
|
||||
self.senders_messages,
|
||||
|
@ -73,6 +70,9 @@ class ReceiverBeacon(Beacon):
|
|||
'dstcall',
|
||||
'receiver_name',
|
||||
'timestamp',
|
||||
|
||||
# 'raw_message',
|
||||
# 'reference_timestamp',
|
||||
|
||||
'version',
|
||||
'platform',
|
||||
|
@ -86,8 +86,6 @@ class ReceiverBeacon(Beacon):
|
|||
'cpu_temp',
|
||||
'senders_visible',
|
||||
'senders_total',
|
||||
# 'rec_crystal_correction',
|
||||
# 'rec_crystal_correction_fine',
|
||||
'rec_input_noise',
|
||||
'senders_signal',
|
||||
'senders_messages',
|
||||
|
@ -103,6 +101,9 @@ class ReceiverBeacon(Beacon):
|
|||
self.dstcall,
|
||||
self.receiver_name,
|
||||
self.timestamp,
|
||||
|
||||
# self.raw_message,
|
||||
# self.reference_timestamp,
|
||||
|
||||
self.version,
|
||||
self.platform,
|
||||
|
@ -116,11 +117,11 @@ class ReceiverBeacon(Beacon):
|
|||
self.cpu_temp,
|
||||
int(self.senders_visible) if self.senders_visible else None,
|
||||
int(self.senders_total) if self.senders_visible else None,
|
||||
# self.rec_crystal_correction,
|
||||
# self.rec_crystal_correction_fine,
|
||||
self.rec_input_noise,
|
||||
self.senders_signal,
|
||||
int(self.senders_messages) if self.senders_messages else None,
|
||||
self.good_senders_signal,
|
||||
int(self.good_senders) if self.good_senders else None,
|
||||
int(self.good_and_bad_senders) if self.good_and_bad_senders else None]
|
||||
|
||||
Index('ix_receiver_beacons_date_receiver_id', func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id)
|
|
@ -12,10 +12,7 @@ class ReceiverStats(Base):
|
|||
|
||||
date = Column(Date)
|
||||
|
||||
# Statistic data
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
aircraft_count = Column(SmallInteger)
|
||||
max_distance = Column(Float)
|
||||
# Static data
|
||||
firstseen = Column(DateTime, index=True)
|
||||
lastseen = Column(DateTime, index=True)
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
|
@ -23,6 +20,19 @@ class ReceiverStats(Base):
|
|||
version = Column(String)
|
||||
platform = Column(String)
|
||||
|
||||
# Statistic data
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
aircraft_count = Column(SmallInteger)
|
||||
max_distance = Column(Float)
|
||||
|
||||
# Ranking data
|
||||
aircraft_beacon_count_ranking_worldwide = Column(SmallInteger)
|
||||
aircraft_beacon_count_ranking_country = Column(SmallInteger)
|
||||
aircraft_count_ranking_worldwide = Column(SmallInteger)
|
||||
aircraft_count_ranking_country = Column(SmallInteger)
|
||||
max_distance_ranking_worldwide = Column(SmallInteger)
|
||||
max_distance_ranking_country = Column(SmallInteger)
|
||||
|
||||
# Relations
|
||||
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
|
||||
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='stats')
|
||||
|
|
28
ogn/utils.py
28
ogn/utils.py
|
@ -5,7 +5,7 @@ from io import StringIO
|
|||
from aerofiles.seeyou import Reader
|
||||
from geopy.exc import GeopyError
|
||||
from geopy.geocoders import Nominatim
|
||||
from ogn.parser.utils import feet2m
|
||||
from ogn.parser.utils import FEETS_TO_METER
|
||||
import requests
|
||||
|
||||
from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location
|
||||
|
@ -89,7 +89,7 @@ def get_airports(cupfile):
|
|||
airport.location_wkt = location.to_wkt()
|
||||
airport.altitude = waypoint['elevation']['value']
|
||||
if (waypoint['elevation']['unit'] == 'ft'):
|
||||
airport.altitude = airport.altitude * feet2m
|
||||
airport.altitude = airport.altitude * FEETS_TO_METER
|
||||
airport.runway_direction = waypoint['runway_direction']
|
||||
airport.runway_length = waypoint['runway_length']['value']
|
||||
if (waypoint['runway_length']['unit'] == 'nm'):
|
||||
|
@ -116,3 +116,27 @@ def open_file(filename):
|
|||
else:
|
||||
f = open(filename, 'rt')
|
||||
return f
|
||||
|
||||
from math import radians, cos, sin, asin, sqrt, atan2, degrees
|
||||
|
||||
def haversine(lat1, lon1, lat2, lon2):
|
||||
"""
|
||||
Calculate the great circle distance between two points
|
||||
on the earth (specified in decimal degrees)
|
||||
"""
|
||||
# convert decimal degrees to radians
|
||||
lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
|
||||
|
||||
# haversine formula
|
||||
dlon = lon2 - lon1
|
||||
dlat = lat2 - lat1
|
||||
a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
|
||||
c = 2 * asin(sqrt(a))
|
||||
r = 6371000.785 # Radius of earth in meters
|
||||
d = c * r
|
||||
|
||||
# calculate bearing
|
||||
bearing = atan2(sin(dlon)*cos(lat2), cos(lat1)*sin(lat2)-sin(lat1)*cos(lat2)*cos(dlon))
|
||||
bearing = (degrees(bearing) + 360) % 360
|
||||
|
||||
return d,bearing
|
5
setup.py
5
setup.py
|
@ -40,10 +40,9 @@ setup(
|
|||
'aerofiles==0.4',
|
||||
'geoalchemy2==0.4.0',
|
||||
'shapely>=1.5.17,<1.6',
|
||||
'ogn-client==0.8.2',
|
||||
'ogn-client==0.9.0',
|
||||
'psycopg2==2.7.3.2',
|
||||
'mgrs==1.3.5',
|
||||
'haversine==0.4.5'
|
||||
'mgrs==1.3.5'
|
||||
],
|
||||
extras_require={
|
||||
'dev': [
|
||||
|
|
|
@ -0,0 +1,101 @@
|
|||
import datetime
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
from ogn.gateway.process_tools import Merger
|
||||
|
||||
|
||||
class MergerTest(unittest.TestCase):
|
||||
def test_different_keys(self):
|
||||
a = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
b = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
c = {'name': 'John', 'receiver_name': 'Observer2', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
d = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 46)}
|
||||
|
||||
callback = MagicMock()
|
||||
merger = Merger(callback=callback)
|
||||
merger.add_message(a)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(b)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(c)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(d)
|
||||
callback.add_message.assert_called_once_with(b)
|
||||
|
||||
merger.flush()
|
||||
calls = [call(a), call(c), call(d)]
|
||||
callback.add_message.assert_has_calls(calls, any_order=True)
|
||||
|
||||
def test_pair(self):
|
||||
a = {'name': 'Jeff', 'receiver_name': 'Observer1','timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': None, 'field_b': 3.141}
|
||||
b = {'name': 'Jeff', 'receiver_name': 'Observer1','timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': 'WTF', 'field_c': None, 'field_d': 1.4142}
|
||||
|
||||
merged = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45), 'field_a': 'WTF', 'field_b': 3.141, 'field_d': 1.4142}
|
||||
|
||||
callback = MagicMock()
|
||||
merger = Merger(callback=callback)
|
||||
merger.add_message(a)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(b)
|
||||
callback.add_message.assert_called_once_with(merged)
|
||||
|
||||
merger.flush()
|
||||
callback.add_message.assert_called_once_with(merged)
|
||||
|
||||
def test_exceed_timedelta(self):
|
||||
a = {'name': 'Jeff', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
b = {'name': 'John', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
c = {'name': 'Fred', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 59)}
|
||||
|
||||
callback = MagicMock()
|
||||
merger = Merger(callback=callback, max_timedelta=datetime.timedelta(seconds=10))
|
||||
merger.add_message(a)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(b)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(c)
|
||||
calls = [call(a), call(b)]
|
||||
callback.add_message.assert_has_calls(calls, any_order=True)
|
||||
|
||||
merger.flush()
|
||||
calls = [call(a), call(b), call(c)]
|
||||
callback.add_message.assert_has_calls(calls, any_order=True)
|
||||
|
||||
def test_exceed_maxlines(self):
|
||||
a = {'name': 'Albert', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 45)}
|
||||
b = {'name': 'Bertram', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 46)}
|
||||
c = {'name': 'Chlodwig', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 47)}
|
||||
d = {'name': 'Dagobert', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 48)}
|
||||
e = {'name': 'Erich', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 49)}
|
||||
f = {'name': 'Frodo', 'receiver_name': 'Observer1', 'timestamp': datetime.datetime(2018, 5, 20, 18, 4, 50)}
|
||||
|
||||
callback = MagicMock()
|
||||
merger = Merger(callback=callback, max_lines=5)
|
||||
merger.add_message(a)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(b)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(c)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(d)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(e)
|
||||
callback.add_message.assert_not_called()
|
||||
|
||||
merger.add_message(f)
|
||||
callback.add_message.assert_called_once_with(a)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
import unittest
|
||||
import unittest.mock as mock
|
||||
|
||||
from ogn.gateway.process import process_beacon, message_to_beacon
|
||||
from ogn.gateway.process import process_raw_message, string_to_message
|
||||
|
||||
|
||||
class ProcessManagerTest(unittest.TestCase):
|
||||
|
@ -16,41 +16,13 @@ class ProcessManagerTest(unittest.TestCase):
|
|||
string1 = "ICA3DD6CD>APRS,qAS,Moosburg:/195919h4820.93N/01151.39EX264/127/A=002204 !W20! id0D3DD6CD -712fpm -0.1rot 8.5dB 0e -2.1kHz gps2x2"
|
||||
string2 = "ICA3DD6CD>APRS,qAS,Moosburg:/195925h4820.90N/01151.07EX263/126/A=002139 !W74! id0D3DD6CD -712fpm +0.0rot 7.8dB 1e -2.1kHz"
|
||||
|
||||
process_beacon(string1)
|
||||
process_raw_message(string1)
|
||||
mock_session.bulk_save_objects.assert_not_called()
|
||||
|
||||
mock_datetime.utcnow.return_value = datetime.datetime(2015, 1, 1, 10, 0, 1) # one second later
|
||||
process_beacon(string2)
|
||||
process_raw_message(string2)
|
||||
self.assertEqual(mock_session.bulk_save_objects.call_count, 1)
|
||||
|
||||
def test_message_to_beacon_brother(self):
|
||||
string1 = "LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:/132457h4849.09NI01708.30E&/A=000528"
|
||||
string2 = "LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:>132457h v0.2.7.arm CPU:0.9 RAM:75.3/253.6MB NTP:2.0ms/-15.2ppm +0.1C 2/2Acfts[1h] RF:+77+1.7ppm/+2.34dB/+6.5dB@10km[5411]/+10.1dB@10km[3/5]"
|
||||
string3 = "BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:/132507h4509.60NI00919.20E&/A=000246"
|
||||
string4 = "BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:>132507h v0.2.7.RPI-GPU CPU:1.2 RAM:35.7/455.2MB NTP:2.5ms/-5.3ppm +67.0C 1/1Acfts[1h] RF:+79+8.8ppm/+4.97dB/-0.0dB@10km[299]/+4.9dB@10km[2/32]"
|
||||
string5 = "Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:/132624h4607.70NI00610.41E&/A=004198 Antenna: chinese, on a pylon, 20 meter above ground"
|
||||
string6 = "Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:>132624h v0.2.7.arm CPU:1.7 RAM:812.3/1022.5MB NTP:1.8ms/+4.5ppm 0.000V 0.000A 3/4Acfts[1h] RF:+67+2.9ppm/+4.18dB/+11.7dB@10km[5018]/+17.2dB@10km[8/16]"
|
||||
|
||||
beacon = message_to_beacon(string1, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNone(beacon)
|
||||
|
||||
beacon = message_to_beacon(string2, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNotNone(beacon)
|
||||
self.assertEqual(beacon.aprs_type, 'merged')
|
||||
|
||||
beacon = message_to_beacon(string3, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNone(beacon)
|
||||
|
||||
beacon = message_to_beacon(string4, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNotNone(beacon)
|
||||
self.assertEqual(beacon.aprs_type, 'merged')
|
||||
|
||||
beacon = message_to_beacon(string5, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNone(beacon)
|
||||
|
||||
beacon = message_to_beacon(string6, reference_date=datetime.date(2015, 1, 1), wait_for_brother=True)
|
||||
self.assertIsNotNone(beacon)
|
||||
self.assertEqual(beacon.aprs_type, 'merged')
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
@ -0,0 +1,42 @@
|
|||
import time
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from ogn.gateway.process_tools import DbSaver, FileSaver
|
||||
|
||||
|
||||
class DbSaverTest(unittest.TestCase):
|
||||
def test(self):
|
||||
a = "Albert"
|
||||
b = "Bertram"
|
||||
c = "Caspar"
|
||||
|
||||
session = MagicMock()
|
||||
saver = DbSaver(session=session)
|
||||
saver.add_message(a)
|
||||
session.bulk_save_objects.assert_not_called()
|
||||
|
||||
saver.add_message(b)
|
||||
session.bulk_save_objects.assert_not_called()
|
||||
|
||||
saver.add_message(c)
|
||||
saver.flush()
|
||||
session.bulk_save_objects.assert_called_once_with([a, b, c])
|
||||
|
||||
def test_timeout(self):
|
||||
a = "Xanthippe"
|
||||
b = "Yvonne"
|
||||
|
||||
session = MagicMock()
|
||||
saver = DbSaver(session=session)
|
||||
saver.add_message(a)
|
||||
session.bulk_save_objects.assert_not_called()
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
saver.add_message(b)
|
||||
session.bulk_save_objects.assert_called_once_with([a, b])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
|
@ -2,7 +2,7 @@ import os
|
|||
import unittest
|
||||
|
||||
from ogn.model import AircraftType
|
||||
from ogn.utils import get_ddb, get_trackable, get_country_code, get_airports
|
||||
from ogn.utils import get_ddb, get_trackable, get_country_code, get_airports, haversine
|
||||
import unittest.mock as mock
|
||||
|
||||
|
||||
|
@ -57,3 +57,8 @@ class TestStringMethods(unittest.TestCase):
|
|||
def test_get_airports(self):
|
||||
airports = get_airports(os.path.dirname(__file__) + '/SeeYou.cup')
|
||||
self.assertGreater(len(airports), 1000)
|
||||
|
||||
def test_haversine(self):
|
||||
distance,bearing = haversine(45.7597, 4.8422, 48.8567, 2.3508)
|
||||
self.assertAlmostEqual(distance, 392216.7, 0)
|
||||
self.assertAlmostEqual(bearing, 332, 0)
|
||||
|
|
Ładowanie…
Reference in New Issue