diff --git a/config/default.py b/config/default.py index 70c924e..45ef1d6 100644 --- a/config/default.py +++ b/config/default.py @@ -24,7 +24,7 @@ CELERYBEAT_SCHEDULE = { 'schedule': timedelta(minutes=5), }, 'update-takeoff-and-landing': { - 'task': 'ogn.collect.takeoff_landing.update_takeoff_landing', + 'task': 'ogn.collect.takeoff_landings.update_takeoff_landings', 'schedule': timedelta(minutes=15), }, 'update-logbook': { diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 46d75bc..3ff0b9a 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -77,64 +77,6 @@ def update_devices(session=None): insert_count = res.rowcount session.commit() - # For each address in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = session.query( - distinct(AircraftBeacon.address).label('address'), - func.first_value(AircraftBeacon.timestamp) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None)], else_=AircraftBeacon.timestamp).asc().nullslast()) - .label('firstseen'), - func.first_value(AircraftBeacon.timestamp) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('lastseen'), - func.first_value(AircraftBeacon.aircraft_type) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('aircraft_type'), - func.first_value(AircraftBeacon.stealth) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('stealth'), - func.first_value(AircraftBeacon.software_version) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('software_version'), - func.first_value(AircraftBeacon.hardware_version) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('hardware_version'), - func.first_value(AircraftBeacon.real_address) - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None)], else_=AircraftBeacon.timestamp).desc().nullslast()) - .label('real_address')) \ - .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ - .subquery() - - update_values = session.query( - Device.address, - case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), - case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), - case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), - (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), - case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), - (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), - case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), - case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), - case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), - (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ - .filter(Device.address == last_valid_values.c.address) \ - .subquery() - - update_receivers = session.query(Device) \ - .filter(Device.address == update_values.c.address) \ - .update({ - Device.firstseen: update_values.c.firstseen, - Device.lastseen: update_values.c.lastseen, - Device.aircraft_type: update_values.c.aircraft_type, - Device.stealth: update_values.c.stealth, - Device.software_version: update_values.c.software_version, - Device.hardware_version: update_values.c.hardware_version, - Device.real_address: update_values.c.real_address}, - synchronize_session='fetch') - # Update relations to aircraft beacons upd = session.query(AircraftBeacon) \ .filter(AircraftBeacon.device_id == null()) \ @@ -170,61 +112,6 @@ def update_receivers(session=None): res = session.execute(ins) insert_count = res.rowcount - # For each name in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = session.query( - distinct(ReceiverBeacon.name).label('name'), - func.first_value(ReceiverBeacon.timestamp) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) - .label('firstseen'), - func.last_value(ReceiverBeacon.timestamp) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast()) - .label('lastseen'), - func.first_value(ReceiverBeacon.location_wkt) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) - .label('location_wkt'), - func.first_value(ReceiverBeacon.altitude) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) - .label('altitude'), - func.first_value(ReceiverBeacon.version) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) - .label('version'), - func.first_value(ReceiverBeacon.platform) - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) - .label('platform')) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .subquery() - - update_values = session.query( - Receiver.name, - case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), - (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), - case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed - (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ - .filter(Receiver.name == last_valid_values.c.name) \ - .subquery() - - update_receivers = session.query(Receiver) \ - .filter(Receiver.name == update_values.c.name) \ - .update({ - Receiver.firstseen: update_values.c.firstseen, - Receiver.lastseen: update_values.c.lastseen, - Receiver.location_wkt: update_values.c.location_wkt, - Receiver.altitude: update_values.c.altitude, - Receiver.version: update_values.c.version, - Receiver.platform: update_values.c.platform, - Receiver.country_code: update_values.c.country_code}, - synchronize_session='fetch') - # Update relations to aircraft beacons update_aircraft_beacons = session.query(AircraftBeacon) \ .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ diff --git a/ogn/collect/stats.py b/ogn/collect/stats.py index 7552a13..9856d42 100644 --- a/ogn/collect/stats.py +++ b/ogn/collect/stats.py @@ -1,12 +1,15 @@ +from datetime import datetime + from celery.utils.log import get_task_logger from sqlalchemy import insert, distinct -from sqlalchemy.sql import null, and_, func -from sqlalchemy.sql.expression import literal_column +from sqlalchemy.sql import null, and_, func, or_ +from sqlalchemy.sql.expression import literal_column, case from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats from .celery import app +from ogn.model.receiver_beacon import ReceiverBeacon logger = get_task_logger(__name__) @@ -27,21 +30,55 @@ def update_device_stats(session=None, date=None): .filter(DeviceStats.date == date) \ .delete() - # Calculate stats for the selected date + # Since "distinct count" does not work in window functions we need a work-around for receiver counting + sq = session.query(AircraftBeacon, + func.dense_rank() + .over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id) + .label('dr')) \ + .filter(and_(AircraftBeacon.device_id != null(), func.date(AircraftBeacon.timestamp) == date)) \ + .filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) \ + .subquery() + + # Calculate stats, firstseen, lastseen and last values != NULL device_stats = session.query( - AircraftBeacon.device_id, - func.date(AircraftBeacon.timestamp).label('date'), - func.count(distinct(AircraftBeacon.receiver_id)).label('receiver_count'), - func.count(AircraftBeacon.id).label('aircraft_beacon_count'), - func.max(AircraftBeacon.altitude).label('max_altitude')) \ - .filter(and_(AircraftBeacon.device_id != null(), AircraftBeacon.receiver_id != null())) \ - .filter(func.date(AircraftBeacon.timestamp) == date) \ - .group_by(AircraftBeacon.device_id, func.date(AircraftBeacon.timestamp)) \ + distinct(sq.c.device_id).label('device_id'), + func.date(sq.c.timestamp).label('date'), + func.max(sq.c.dr) + .over(partition_by=sq.c.device_id) + .label('receiver_count'), + func.max(sq.c.altitude) + .over(partition_by=sq.c.device_id) + .label('max_altitude'), + func.count(sq.c.id) + .over(partition_by=sq.c.device_id) + .label('aircraft_beacon_count'), + func.first_value(sq.c.timestamp) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()) + .label('firstseen'), + func.first_value(sq.c.timestamp) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('lastseen'), + func.first_value(sq.c.aircraft_type) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.aircraft_type == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('aircraft_type'), + func.first_value(sq.c.stealth) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.stealth == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('stealth'), + func.first_value(sq.c.software_version) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.software_version == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('software_version'), + func.first_value(sq.c.hardware_version) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.hardware_version == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('hardware_version'), + func.first_value(sq.c.real_address) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.real_address == null(), None)], else_=sq.c.timestamp).desc().nullslast()) + .label('real_address')) \ .subquery() # And insert them ins = insert(DeviceStats).from_select( - [DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.aircraft_beacon_count, DeviceStats.max_altitude], + [DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.max_altitude, DeviceStats.aircraft_beacon_count, DeviceStats.firstseen, DeviceStats.lastseen, DeviceStats.aircraft_type, DeviceStats.stealth, + DeviceStats.software_version, DeviceStats.hardware_version, DeviceStats.real_address], device_stats) res = session.execute(ins) insert_counter = res.rowcount diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landings.py similarity index 91% rename from ogn/collect/takeoff_landing.py rename to ogn/collect/takeoff_landings.py index 594675b..5d376f2 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landings.py @@ -44,7 +44,6 @@ def update_takeoff_landings(session=None): # make a query with current, previous and next position beacon_selection = session.query(AircraftBeacon.id) \ - .filter(AircraftBeacon.status == null()) \ .order_by(AircraftBeacon.timestamp) \ .limit(1000000) \ .subquery() @@ -71,7 +70,6 @@ def update_takeoff_landings(session=None): AircraftBeacon.altitude, func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'), func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next')) \ - .filter(AircraftBeacon.status == null()) \ .filter(AircraftBeacon.id == beacon_selection.c.id) \ .subquery() @@ -133,13 +131,7 @@ def update_takeoff_landings(session=None): result = session.execute(ins) counter = result.rowcount - # mark the computated AircraftBeacons as 'used' - update_aircraft_beacons = session.query(AircraftBeacon) \ - .filter(AircraftBeacon.id == sq2.c.id) \ - .update({AircraftBeacon.status: 1}, - synchronize_session='fetch') - session.commit() - logger.debug("Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(counter, update_aircraft_beacons)) + logger.debug("Inserted {} TakeoffLandings".format(counter)) - return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(counter, update_aircraft_beacons) + return "Inserted {} TakeoffLandings".format(counter) diff --git a/ogn/commands/bulkimport.py b/ogn/commands/bulkimport.py index fc4ba6c..923987f 100644 --- a/ogn/commands/bulkimport.py +++ b/ogn/commands/bulkimport.py @@ -116,7 +116,6 @@ def drop_indices(): DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id; DROP INDEX IF EXISTS ix_aircraft_beacons_device_id; DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp; - DROP INDEX IF EXISTS ix_aircraft_beacons_status; """) print("Dropped indices of AircraftBeacon") @@ -135,7 +134,6 @@ def create_indices(): CREATE INDEX ix_aircraft_beacon_receiver_id ON aircraft_beacons USING BTREE(receiver_id); CREATE INDEX ix_aircraft_beacon_device_id ON aircraft_beacons USING BTREE(device_id); CREATE INDEX ix_aircraft_beacon_timestamp ON aircraft_beacons USING BTREE(timestamp); - CREATE INDEX ix_aircraft_beacon_status ON aircraft_beacons USING BTREE(status); """) print("Created indices for AircraftBeacon") @@ -215,6 +213,7 @@ def import_aircraft_beacon_logfile(csv_logfile): altitude integer, name character varying, receiver_name character varying(9), + dstcall character varying, "timestamp" timestamp without time zone, track integer, ground_speed double precision, @@ -233,7 +232,9 @@ def import_aircraft_beacon_logfile(csv_logfile): software_version double precision, hardware_version smallint, real_address character varying(6), - signal_power double precision + signal_power double precision, + + location_mgrs character varying(15) ); """ @@ -279,12 +280,12 @@ def import_aircraft_beacon_logfile(csv_logfile): print("Inserted missing Receivers") session.execute(""" - INSERT INTO aircraft_beacons(location, altitude, name, receiver_name, timestamp, track, ground_speed, + INSERT INTO aircraft_beacons(location, altitude, name, receiver_name, dstcall, timestamp, track, ground_speed, address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power, - status, receiver_id, device_id) - SELECT t.location, t.altitude, t.name, t.receiver_name, t.timestamp, t.track, t.ground_speed, + receiver_id, device_id) + SELECT t.location, t.altitude, t.name, t.receiver_name, t.dstcall, t.timestamp, t.track, t.ground_speed, t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power, - 0, r.id, d.id + r.id, d.id FROM aircraft_beacons_temp t, receivers r, devices d WHERE t.receiver_name = r.name AND t.address = d.address """) @@ -307,6 +308,7 @@ def import_receiver_beacon_logfile(csv_logfile): altitude integer, name character varying, receiver_name character varying(9), + dstcall character varying, "timestamp" timestamp without time zone, track integer, ground_speed double precision, @@ -365,12 +367,12 @@ def import_receiver_beacon_logfile(csv_logfile): print("Inserted missing Receivers") session.execute(""" - INSERT INTO receiver_beacons(location, altitude, name, receiver_name, timestamp, track, ground_speed, + INSERT INTO receiver_beacons(location, altitude, name, receiver_name, dstcall, timestamp, track, ground_speed, version, platform, cpu_load, free_ram, total_ram, ntp_error, rt_crystal_correction, voltage,amperage, cpu_temp, senders_visible, senders_total, rec_input_noise, senders_signal, senders_messages, good_senders_signal, good_senders, good_and_bad_senders, - status, receiver_id) - SELECT t.location, t.altitude, t.name, t.receiver_name, t.timestamp, t.track, t.ground_speed, + receiver_id) + SELECT t.location, t.altitude, t.name, t.receiver_name, t.dstcall, t.timestamp, t.track, t.ground_speed, t.version, t.platform, t.cpu_load, t.free_ram, t.total_ram, t.ntp_error, t.rt_crystal_correction, t.voltage,amperage, t.cpu_temp, t.senders_visible, t.senders_total, t.rec_input_noise, t.senders_signal, t.senders_messages, t.good_senders_signal, t.good_senders, t.good_and_bad_senders, - 0, r.id + r.id FROM receiver_beacons_temp t, receivers r WHERE t.name = r.name """) diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index 81742e3..de8f74a 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -4,7 +4,7 @@ from datetime import timedelta, datetime from manager import Manager from ogn.collect.logbook import update_logbook -from ogn.collect.takeoff_landing import update_takeoff_landings +from ogn.collect.takeoff_landings import update_takeoff_landings from ogn.commands.dbutils import session from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook from sqlalchemy import and_, or_ @@ -19,7 +19,7 @@ manager = Manager() def compute_takeoff_landing(): """Compute takeoffs and landings.""" print("Compute takeoffs and landings...") - result = update_takeoff_landing.delay() + result = update_takeoff_landings.delay() counter = result.get() print("New takeoffs/landings: {}".format(counter)) diff --git a/ogn/gateway/process.py b/ogn/gateway/process.py index 4366730..004118d 100644 --- a/ogn/gateway/process.py +++ b/ogn/gateway/process.py @@ -13,9 +13,12 @@ myMGRS = MGRS() def replace_lonlat_with_wkt(message): - location = Location(message['longitude'], message['latitude']) + latitude = message['latitude'] + longitude = message['longitude'] + + location = Location(longitude, latitude) message['location_wkt'] = location.to_wkt() - message['location_mgrs'] = myMGRS.toMGRS(message['latitude'], message['longitude']) + message['location_mgrs'] = myMGRS.toMGRS(latitude, longitude).decode('utf-8') del message['latitude'] del message['longitude'] return message @@ -44,6 +47,9 @@ def message_to_beacon(raw_message, reference_date): logger.error('Drop packet, {}'.format(e.message)) except TypeError as e: logger.error('TypeError: {}'.format(raw_message)) + except Exception as e: + logger.error(raw_message) + logger.error(e) return beacon diff --git a/ogn/model/aircraft_beacon.py b/ogn/model/aircraft_beacon.py index 810c08b..61fb0e7 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn/model/aircraft_beacon.py @@ -9,7 +9,7 @@ class AircraftBeacon(Beacon): # Flarm specific data address_type = Column(SmallInteger) - aircraft_type = Column(SmallInteger, index=True) + aircraft_type = Column(SmallInteger) stealth = Column(Boolean) address = Column(String(6)) climb_rate = Column(Float) @@ -36,8 +36,6 @@ class AircraftBeacon(Beacon): noise_level = None relays = None - status = Column(SmallInteger, index=True) - # Calculated values distance = Column(Float) location_mgrs = Column(String(15), index=True) @@ -71,7 +69,7 @@ class AircraftBeacon(Beacon): self.real_address, self.signal_power, - self.status) + self.location_mgrs) @classmethod def get_csv_columns(self): @@ -79,6 +77,7 @@ class AircraftBeacon(Beacon): 'altitude', 'name', 'receiver_name', + 'dstcall', 'timestamp', 'track', 'ground_speed', @@ -97,14 +96,17 @@ class AircraftBeacon(Beacon): 'software_version', 'hardware_version', 'real_address', - 'signal_power'] + 'signal_power', + + 'location_mgrs'] def get_csv_values(self): return [ self.location_wkt, - int(self.altitude) if self.altitude else None, + int(self.altitude), self.name, self.receiver_name, + self.dstcall, self.timestamp, self.track, self.ground_speed, @@ -117,10 +119,12 @@ class AircraftBeacon(Beacon): self.turn_rate, self.flightlevel, self.signal_quality, - int(self.error_count) if self.error_count else None, + self.error_count, self.frequency_offset, self.gps_status, self.software_version, self.hardware_version, self.real_address, - self.signal_power] + self.signal_power, + + self.location_mgrs] diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index 6ac44f7..6b3e2cc 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -16,7 +16,7 @@ class Beacon(AbstractConcreteBase, Base): name = Column(String) receiver_name = Column(String(9)) - dstcall = None + dstcall = Column(String) timestamp = Column(DateTime, index=True) symboltable = None symbolcode = None diff --git a/ogn/model/device.py b/ogn/model/device.py index aa9a98b..d33b9b3 100644 --- a/ogn/model/device.py +++ b/ogn/model/device.py @@ -8,6 +8,7 @@ class Device(Base): __tablename__ = 'devices' id = Column(Integer, primary_key=True) + address = Column(String(6), index=True) firstseen = Column(DateTime, index=True) lastseen = Column(DateTime, index=True) diff --git a/ogn/model/device_stats.py b/ogn/model/device_stats.py index ff26bf3..af3d8eb 100644 --- a/ogn/model/device_stats.py +++ b/ogn/model/device_stats.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, Date, Float, ForeignKey +from sqlalchemy import Column, Integer, Date, DateTime, Float, ForeignKey, SmallInteger, Boolean, String from sqlalchemy.orm import relationship from .base import Base @@ -10,9 +10,16 @@ class DeviceStats(Base): id = Column(Integer, primary_key=True) date = Column(Date) + max_altitude = Column(Float) receiver_count = Column(Integer) aircraft_beacon_count = Column(Integer) - max_altitude = Column(Float) + firstseen = Column(DateTime) + lastseen = Column(DateTime) + aircraft_type = Column(SmallInteger) + stealth = Column(Boolean) + software_version = Column(Float) + hardware_version = Column(SmallInteger) + real_address = Column(String(6)) # Relations device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) diff --git a/ogn/model/receiver_beacon.py b/ogn/model/receiver_beacon.py index ee47986..ba6fa39 100644 --- a/ogn/model/receiver_beacon.py +++ b/ogn/model/receiver_beacon.py @@ -31,8 +31,6 @@ class ReceiverBeacon(Beacon): user_comment = None - status = Column(SmallInteger, index=True) - # Relations receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL')) receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_beacons') @@ -61,9 +59,7 @@ class ReceiverBeacon(Beacon): self.senders_messages, self.good_senders_signal, self.good_senders, - self.good_and_bad_senders, - - self.status) + self.good_and_bad_senders) @classmethod def get_csv_columns(self): @@ -71,6 +67,7 @@ class ReceiverBeacon(Beacon): 'altitude', 'name', 'receiver_name', + 'dstcall', 'timestamp', 'track', 'ground_speed', @@ -102,6 +99,7 @@ class ReceiverBeacon(Beacon): int(self.altitude) if self.altitude else None, self.name, self.receiver_name, + self.dstcall, self.timestamp, self.track, self.ground_speed, diff --git a/ogn/model/takeoff_landing.py b/ogn/model/takeoff_landing.py index d65d04f..621e491 100644 --- a/ogn/model/takeoff_landing.py +++ b/ogn/model/takeoff_landing.py @@ -7,15 +7,13 @@ from .base import Base class TakeoffLanding(Base): __tablename__ = 'takeoff_landings' - id = Column(Integer, primary_key=True) + device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True) + airport_id = Column(Integer, ForeignKey('airports.id', ondelete='SET NULL'), primary_key=True) + timestamp = Column(DateTime, primary_key=True) is_takeoff = Column(Boolean) - timestamp = Column(DateTime, index=True) track = Column(Integer) # Relations - airport_id = Column(Integer, ForeignKey('airports.id', ondelete='SET NULL'), index=True) airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings') - - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) device = relationship('Device', foreign_keys=[device_id], backref='takeoff_landings') diff --git a/tests/collect/test_database.py b/tests/collect/test_database.py index bfa8c12..e72c7a4 100644 --- a/tests/collect/test_database.py +++ b/tests/collect/test_database.py @@ -19,18 +19,6 @@ class TestDB(unittest.TestCase): from ogn.commands.database import init init() - # Prepare Beacons - self.ab01 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:00', aircraft_type=1, stealth=False, error_count=0, software_version=None, hardware_version=None, real_address=None) - self.ab02 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:01', aircraft_type=1, stealth=False, error_count=0, software_version=6.01, hardware_version=None, real_address=None) - self.ab03 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:02', aircraft_type=1, stealth=False, error_count=1, software_version=6.02, hardware_version=None, real_address=None) - self.ab04 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:03', aircraft_type=1, stealth=False, error_count=0, software_version=None, hardware_version=5, real_address='DD1234') - self.ab05 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:04', aircraft_type=1, stealth=False, error_count=0, software_version=6.00, hardware_version=123, real_address='DDxxxx') - self.ab06 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:05', aircraft_type=1, stealth=False, error_count=0, software_version=None, hardware_version=None, real_address='DD0815') - - self.rb01 = ReceiverBeacon(name='Koenigsdf', timestamp='2017-12-10 09:55:00', altitude=601, version='0.2.5', platform='ARM') - self.rb02 = ReceiverBeacon(name='Koenigsdf', timestamp='2017-12-10 10:00:00', altitude=601, version='0.2.7', platform='ARM') - self.rb03 = ReceiverBeacon(name='Koenigsdf', timestamp='2017-12-10 10:05:00', altitude=601, version='0.2.6', platform='ARM') - def tearDown(self): session = self.session session.execute("DELETE FROM device_infos") @@ -43,80 +31,25 @@ class TestDB(unittest.TestCase): def test_update_devices(self): session = self.session - # Compute 1st beacon - session.add(self.ab01) - session.commit() + ab01 = AircraftBeacon(receiver_name='Koenigsdf', address='DD4711', timestamp='2017-12-10 10:00:00') + rb01 = ReceiverBeacon(name='Bene', timestamp='2017-12-10 09:59:50') + d01 = Device(address='DD4711') + r01 = Receiver(name='Koenigsdf') + session.bulk_save_objects([ab01, rb01, d01, r01]) update_devices(session) - - devices = session.query(Device).all() - self.assertEqual(len(devices), 1) - self.assertEqual(devices[0].address, 'DD4711') - self.assertEqual(devices[0].software_version, None) - self.assertEqual(self.ab01.device_id, devices[0].id) - - # Compute 2nd beacon: changed software version - session.add(self.ab02) - session.commit() - - update_devices(session) - devices = session.query(Device).all() - self.assertEqual(len(devices), 1) - self.assertEqual(devices[0].address, 'DD4711') - self.assertEqual(devices[0].software_version, 6.01) - self.assertEqual(self.ab02.device_id, devices[0].id) - - # Compute 3rd beacon: changed software version, but with error_count > 0 - session.add(self.ab03) - session.commit() - - update_devices(session) - devices = session.query(Device).all() - self.assertEqual(len(devices), 1) - self.assertEqual(devices[0].address, 'DD4711') - self.assertEqual(devices[0].software_version, 6.01) - self.assertEqual(devices[0].hardware_version, None) - self.assertEqual(devices[0].real_address, None) - self.assertEqual(self.ab03.device_id, devices[0].id) - - # Compute 4.-6. beacon - session.add(self.ab04) - session.add(self.ab06) # order is not important - session.add(self.ab05) - session.commit() - - update_devices(session) - devices = session.query(Device).all() - self.assertEqual(len(devices), 1) - self.assertEqual(devices[0].address, 'DD4711') - self.assertEqual(devices[0].software_version, 6.0) - self.assertEqual(devices[0].hardware_version, 123) - self.assertEqual(devices[0].real_address, 'DD0815') - self.assertEqual(self.ab04.device_id, devices[0].id) - self.assertEqual(self.ab05.device_id, devices[0].id) - self.assertEqual(self.ab06.device_id, devices[0].id) - - def test_update_receivers(self): - session = self.session - - # Compute beacons - session.add(self.rb01) - session.add(self.rb02) - session.add(self.rb03) - session.add(self.ab01) - session.commit() - update_receivers(session) - receivers = session.query(Receiver).all() - self.assertEqual(len(receivers), 1) - self.assertEqual(receivers[0].name, 'Koenigsdf') - self.assertEqual(receivers[0].altitude, 601) - self.assertEqual(receivers[0].version, '0.2.6') - self.assertEqual(self.rb01.receiver_id, receivers[0].id) - self.assertEqual(self.rb02.receiver_id, receivers[0].id) - self.assertEqual(self.rb03.receiver_id, receivers[0].id) - self.assertEqual(self.ab01.receiver_id, receivers[0].id) + aircraft_beacons = session.query(AircraftBeacon).all() + self.assertEqual(len(aircraft_beacons), 1) + aircraft_beacon = aircraft_beacons[0] + self.assertEqual(aircraft_beacon.device.address, 'DD4711') + self.assertEqual(aircraft_beacon.receiver.name, 'Koenigsdf') + + receiver_beacons = session.query(ReceiverBeacon).all() + self.assertEqual(len(receiver_beacons), 1) + receiver_beacon = receiver_beacons[0] + self.assertEqual(receiver_beacon.receiver.name, 'Bene') def test_import_ddb_file(self): session = self.session diff --git a/tests/collect/test_stats.py b/tests/collect/test_stats.py index f2812ca..7dc1ba1 100644 --- a/tests/collect/test_stats.py +++ b/tests/collect/test_stats.py @@ -1,7 +1,8 @@ import unittest import os +from datetime import datetime -from ogn.model import DeviceStats +from ogn.model import AircraftBeacon, ReceiverBeacon, Receiver, Device, DeviceStats from ogn.collect.stats import update_device_stats @@ -20,38 +21,197 @@ class TestDB(unittest.TestCase): from ogn.commands.database import init init() - session.execute("INSERT INTO devices(address) VALUES('DDEFF7')") - session.execute("INSERT INTO receivers(name) VALUES('Koenigsdf')") - session.execute("INSERT INTO receivers(name) VALUES('Ohlstadt')") + # Prepare Beacons + self.ab01 = AircraftBeacon(timestamp='2017-12-10 10:00:01') + self.ab02 = AircraftBeacon(timestamp='2017-12-10 10:00:02') + self.ab03 = AircraftBeacon(timestamp='2017-12-10 10:00:03') + self.ab04 = AircraftBeacon(timestamp='2017-12-10 10:00:04') + self.ab05 = AircraftBeacon(timestamp='2017-12-10 10:00:05') + self.ab06 = AircraftBeacon(timestamp='2017-12-10 10:00:05') + + self.rb01 = ReceiverBeacon(timestamp='2017-12-10 09:55:00', altitude=601, version='0.2.5', platform='ARM') + self.rb02 = ReceiverBeacon(timestamp='2017-12-10 10:00:00', altitude=601, version='0.2.7', platform='ARM') + self.rb03 = ReceiverBeacon(timestamp='2017-12-10 10:05:00', altitude=601, version='0.2.6', platform='ARM') + + self.r01 = Receiver(name='Koenigsdf') + self.r02 = Receiver(name='Bene') + + self.d01 = Device(address='DD4711') + + session.add(self.r01) + session.add(self.d01) + session.commit() def tearDown(self): session = self.session - session.execute("DELETE FROM aircraft_beacons") - session.execute("DELETE FROM devices") - + session.execute("DELETE FROM device_infos") session.execute("DELETE FROM device_stats") - session.execute("DELETE FROM receiver_stats") + session.execute("DELETE FROM devices") + session.execute("DELETE FROM receivers") + session.execute("DELETE FROM aircraft_beacons") + session.execute("DELETE FROM receiver_beacons") + session.commit() - def test_update_device_stats(self): + def test_update_devices(self): session = self.session - session.execute("INSERT INTO aircraft_beacons(address, receiver_name, altitude, timestamp) VALUES('DDEFF7','Koenigsdf',604,'2016-07-02 10:47:12')") - session.execute("INSERT INTO aircraft_beacons(address, receiver_name, altitude, timestamp) VALUES('DDEFF7','Koenigsdf',605,'2016-07-02 10:47:32')") - session.execute("INSERT INTO aircraft_beacons(address, receiver_name, altitude, timestamp) VALUES('DDEFF7','Koenigsdf',606,'2016-07-02 10:47:52')") - session.execute("INSERT INTO aircraft_beacons(address, receiver_name, altitude, timestamp) VALUES('DDEFF7','Ohlstadt',606,'2016-07-02 10:48:12')") - session.execute("INSERT INTO aircraft_beacons(address, receiver_name, altitude, timestamp) VALUES('DDEFF7','Ohlstadt',606,'2016-07-02 10:48:24')") + # Compute 1st beacon + self.ab01.device = self.d01 + self.ab01.receiver = self.r01 + session.add(self.ab01) + session.commit() - session.execute("UPDATE aircraft_beacons SET device_id = d.id, receiver_id = r.id FROM devices d, receivers r WHERE aircraft_beacons.address=d.address AND aircraft_beacons.receiver_name=r.name") + update_device_stats(session, date='2017-12-10') - update_device_stats(session, date='2016-07-02') - stats = session.query(DeviceStats).all() + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) - self.assertEqual(len(stats), 1) + self.assertEqual(devicestats[0].max_altitude, None) + self.assertEqual(devicestats[0].receiver_count, 1) + self.assertEqual(devicestats[0].aircraft_beacon_count, 1) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].aircraft_type, None) + self.assertEqual(devicestats[0].stealth, None) + self.assertEqual(devicestats[0].software_version, None) + self.assertEqual(devicestats[0].hardware_version, None) + self.assertEqual(devicestats[0].real_address, None) - stat = stats[0] - self.assertEqual(stat.receiver_count, 2) - self.assertEqual(stat.aircraft_beacon_count, 5) - self.assertEqual(stat.max_altitude, 606) + # Compute 2nd beacon: set altitude, aircraft_type and stealth + self.ab02.device = self.d01 + self.ab02.receiver = self.r01 + self.ab02.altitude = 200 + self.ab02.aircraft_type = 3 + self.ab02.stealth = False + session.add(self.ab02) + session.commit() + + update_device_stats(session, date='2017-12-10') + + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) + + self.assertEqual(devicestats[0].max_altitude, 200) + self.assertEqual(devicestats[0].receiver_count, 1) + self.assertEqual(devicestats[0].aircraft_beacon_count, 2) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 2)) + self.assertEqual(devicestats[0].aircraft_type, 3) + self.assertEqual(devicestats[0].stealth, False) + self.assertEqual(devicestats[0].software_version, None) + self.assertEqual(devicestats[0].hardware_version, None) + self.assertEqual(devicestats[0].real_address, None) + + # Compute 3rd beacon: changed software version, but with error_count > 0 + self.ab03.device = self.d01 + self.ab03.receiver = self.r01 + self.ab03.error_count = 1 + self.ab03.software_version = 6.01 + session.add(self.ab03) + session.commit() + + update_device_stats(session, date='2017-12-10') + + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) + + self.assertEqual(devicestats[0].max_altitude, 200) + self.assertEqual(devicestats[0].receiver_count, 1) + self.assertEqual(devicestats[0].aircraft_beacon_count, 2) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 2)) + self.assertEqual(devicestats[0].aircraft_type, 3) + self.assertEqual(devicestats[0].stealth, False) + self.assertEqual(devicestats[0].software_version, None) + self.assertEqual(devicestats[0].hardware_version, None) + self.assertEqual(devicestats[0].real_address, None) + + # Compute 4. beacon: another receiver, greater altitude, software_version, hardware_version, real_address + self.ab04.device = self.d01 + self.ab04.receiver = self.r02 + self.ab04.altitude = 250 + self.ab04.software_version = 6.01 + self.ab04.hardware_version = 15 + self.ab04.real_address = 'DDALFA' + session.add(self.ab04) + session.commit() + + update_device_stats(session, date='2017-12-10') + + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) + + self.assertEqual(devicestats[0].max_altitude, 250) + self.assertEqual(devicestats[0].receiver_count, 2) + self.assertEqual(devicestats[0].aircraft_beacon_count, 3) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 4)) + self.assertEqual(devicestats[0].aircraft_type, 3) + self.assertEqual(devicestats[0].stealth, False) + self.assertEqual(devicestats[0].software_version, 6.01) + self.assertEqual(devicestats[0].hardware_version, 15) + self.assertEqual(devicestats[0].real_address, 'DDALFA') + + # Compute 5. beacon: lower altitude, stealth + self.ab05.device = self.d01 + self.ab05.receiver = self.r02 + self.ab05.altitude = 100 + self.ab05.stealth = True + session.add(self.ab05) + session.commit() + + update_device_stats(session, date='2017-12-10') + + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) + + self.assertEqual(devicestats[0].max_altitude, 250) + self.assertEqual(devicestats[0].receiver_count, 2) + self.assertEqual(devicestats[0].aircraft_beacon_count, 4) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 10, 0, 1)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 5)) + self.assertEqual(devicestats[0].aircraft_type, 3) + self.assertEqual(devicestats[0].stealth, True) + self.assertEqual(devicestats[0].software_version, 6.01) + self.assertEqual(devicestats[0].hardware_version, 15) + self.assertEqual(devicestats[0].real_address, 'DDALFA') + + # Compute 6. beacon: beacon from past, greater altitude, newer version + self.ab06.device = self.d01 + self.ab06.receiver = self.r02 + self.ab06.timestamp = datetime(2017, 12, 10, 9, 59, 50) + self.ab06.altitude = 300 + self.ab06.software_version = 6.02 + session.add(self.ab06) + session.commit() + + update_device_stats(session, date='2017-12-10') + + devicestats = session.query(DeviceStats).all() + self.assertEqual(len(devicestats), 1) + self.assertEqual(devicestats[0].device, self.d01) + + self.assertEqual(devicestats[0].max_altitude, 300) + self.assertEqual(devicestats[0].receiver_count, 2) + self.assertEqual(devicestats[0].aircraft_beacon_count, 5) + self.assertEqual(devicestats[0].date, datetime.strptime('2017-12-10', '%Y-%m-%d').date()) + self.assertEqual(devicestats[0].firstseen, datetime(2017, 12, 10, 9, 59, 50)) + self.assertEqual(devicestats[0].lastseen, datetime(2017, 12, 10, 10, 0, 5)) + self.assertEqual(devicestats[0].aircraft_type, 3) + self.assertEqual(devicestats[0].stealth, True) + self.assertEqual(devicestats[0].software_version, 6.01) + self.assertEqual(devicestats[0].hardware_version, 15) + self.assertEqual(devicestats[0].real_address, 'DDALFA') if __name__ == '__main__': diff --git a/tests/collect/test_takeoff_landing.py b/tests/collect/test_takeoff_landing.py index 631d802..990e06e 100644 --- a/tests/collect/test_takeoff_landing.py +++ b/tests/collect/test_takeoff_landing.py @@ -3,7 +3,7 @@ import os from ogn.model import TakeoffLanding -from ogn.collect.takeoff_landing import update_takeoff_landings +from ogn.collect.takeoff_landings import update_takeoff_landings class TestDB(unittest.TestCase): @@ -40,7 +40,7 @@ class TestDB(unittest.TestCase): i = 0 for takeoff_landing in query.all(): i = i + 1 - print("{} {} {} {} {} {}".format(takeoff_landing.id, takeoff_landing.device_id, takeoff_landing.airport_id, takeoff_landing.timestamp, takeoff_landing.is_takeoff, takeoff_landing.track)) + print("{} {} {} {} {}".format(takeoff_landing.device_id, takeoff_landing.airport_id, takeoff_landing.timestamp, takeoff_landing.is_takeoff, takeoff_landing.track)) return i