diff --git a/ogn/collect/stats.py b/ogn/collect/stats.py index 9856d42..bf91ab1 100644 --- a/ogn/collect/stats.py +++ b/ogn/collect/stats.py @@ -3,7 +3,7 @@ from datetime import datetime from celery.utils.log import get_task_logger from sqlalchemy import insert, distinct -from sqlalchemy.sql import null, and_, func, or_ +from sqlalchemy.sql import null, and_, func, or_, update from sqlalchemy.sql.expression import literal_column, case from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats @@ -35,7 +35,7 @@ def update_device_stats(session=None, date=None): func.dense_rank() .over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id) .label('dr')) \ - .filter(and_(AircraftBeacon.device_id != null(), func.date(AircraftBeacon.timestamp) == date)) \ + .filter(and_(func.date(AircraftBeacon.timestamp) == date, AircraftBeacon.device_id != null())) \ .filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) \ .subquery() @@ -89,9 +89,12 @@ def update_device_stats(session=None, date=None): @app.task -def update_receiver_stats(date=None): +def update_receiver_stats(session=None, date=None): """Add/update receiver stats.""" + if session is None: + session = app.session + if not date: logger.warn("A date is needed for calculating stats. Exiting") return None @@ -101,25 +104,49 @@ def update_receiver_stats(date=None): .filter(ReceiverStats.date == date) \ .delete() - # Calculate stats for the selected date + # Calculate stats, firstseen, lastseen and last values != NULL receiver_stats = session.query( - AircraftBeacon.receiver_id, - literal_column("'{}'".format(date)).label('date'), - func.count(AircraftBeacon.id).label('aircraft_beacon_count'), - func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'), - func.max(AircraftBeacon.distance).label('max_distance')) \ - .filter(AircraftBeacon.receiver_id != null()) \ - .filter(func.date(AircraftBeacon.timestamp) == date) \ - .group_by(AircraftBeacon.receiver_id) \ + distinct(ReceiverBeacon.receiver_id).label('receiver_id'), + func.date(ReceiverBeacon.timestamp).label('date'), + func.first_value(ReceiverBeacon.timestamp) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast()) + .label('firstseen'), + func.first_value(ReceiverBeacon.timestamp) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) + .label('lastseen'), + func.first_value(ReceiverBeacon.location_wkt) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) + .label('location_wkt'), + func.first_value(ReceiverBeacon.altitude) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) + .label('altitude'), + func.first_value(ReceiverBeacon.version) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) + .label('version'), + func.first_value(ReceiverBeacon.platform) + .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast()) + .label('platform')) \ .subquery() # And insert them ins = insert(ReceiverStats).from_select( - [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance], + [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.firstseen, ReceiverStats.lastseen, ReceiverStats.location_wkt, ReceiverStats.altitude, ReceiverStats.version, ReceiverStats.platform], receiver_stats) res = session.execute(ins) insert_counter = res.rowcount session.commit() - logger.debug("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) + logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) - return "ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter) + # Update AircraftBeacon distances + upd = update(AircraftBeacon) \ + .where(and_(func.date(AircraftBeacon.timestamp) == ReceiverStats.date, + AircraftBeacon.receiver_id == ReceiverStats.receiver_id, + AircraftBeacon.distance == null())) \ + .values({"distance": func.ST_Distance_Sphere(AircraftBeacon.location_wkt, ReceiverStats.location_wkt)}) + + result = session.execute(upd) + update_counter = result.rowcount + session.commit() + logger.warn("Updated {} AircraftBeacons".format(update_counter)) + + return "ReceiverStats for {}: {} deleted, {} inserted, AircraftBeacons: {} updated".format(date, deleted_counter, insert_counter, update_counter) diff --git a/ogn/collect/takeoff_landings.py b/ogn/collect/takeoff_landings.py index 5d376f2..d9fb158 100644 --- a/ogn/collect/takeoff_landings.py +++ b/ogn/collect/takeoff_landings.py @@ -1,8 +1,8 @@ -from datetime import timedelta +from datetime import datetime, timedelta from celery.utils.log import get_task_logger -from sqlalchemy import and_, or_, insert, between, exists +from sqlalchemy import and_, or_, insert, update, between, exists from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import case @@ -13,7 +13,7 @@ logger = get_task_logger(__name__) @app.task -def update_takeoff_landings(session=None): +def update_takeoff_landings(session=None, date=None): """Compute takeoffs and landings.""" logger.info("Compute takeoffs and landings.") @@ -43,10 +43,20 @@ def update_takeoff_landings(session=None): AircraftBeacon.receiver_id) # make a query with current, previous and next position - beacon_selection = session.query(AircraftBeacon.id) \ - .order_by(AircraftBeacon.timestamp) \ - .limit(1000000) \ - .subquery() + if date is None: + beacon_selection = session.query(AircraftBeacon.id) \ + .filter(AircraftBeacon.status == 0) \ + .order_by(AircraftBeacon.timestamp) \ + .subquery() + else: + my_day = datetime.strptime(date, '%Y-%m-%d') + beacon_selection = session.query(AircraftBeacon.id) \ + .filter(and_(AircraftBeacon.status == 0, + AircraftBeacon.timestamp >= my_day - timedelta(minutes=5), + AircraftBeacon.timestamp < my_day + timedelta(days=1, minutes=5))) \ + .order_by(AircraftBeacon.timestamp) \ + .limit(100000) \ + .subquery() sq = session.query( AircraftBeacon.id, @@ -78,6 +88,9 @@ def update_takeoff_landings(session=None): .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ .subquery() + logger.warn(sq2) + return + # find possible takeoffs and landings sq3 = session.query( sq2.c.id, @@ -129,9 +142,17 @@ def update_takeoff_landings(session=None): TakeoffLanding.airport_id), takeoff_landing_query) result = session.execute(ins) - counter = result.rowcount + insert_counter = result.rowcount + logger.warn("Inserted {} TakeoffLandings".format(insert_counter)) + # Set calculated beacons as 'used' + upd = update(AircraftBeacon) \ + .where(AircraftBeacon.id == sq2.c.id) \ + .values({"status": 1}) + + result = session.execute(upd) + update_counter = result.rowcount session.commit() - logger.debug("Inserted {} TakeoffLandings".format(counter)) + logger.warn("Updated {} AircraftBeacons".format(update_counter)) - return "Inserted {} TakeoffLandings".format(counter) + return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(insert_counter, update_counter) diff --git a/ogn/commands/bulkimport.py b/ogn/commands/bulkimport.py index 923987f..ea6c459 100644 --- a/ogn/commands/bulkimport.py +++ b/ogn/commands/bulkimport.py @@ -112,9 +112,11 @@ def convert(sourcefile, path=''): def drop_indices(): """Drop indices of AircraftBeacon.""" session.execute(""" - DROP INDEX IF EXISTS idx_aircraft_beacons_location; - DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id; - DROP INDEX IF EXISTS ix_aircraft_beacons_device_id; + DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name; + DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address; + DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp; + DROP INDEX IF EXISTS ix_aircraft_beacons_location; + DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs; DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp; """) print("Dropped indices of AircraftBeacon") @@ -130,10 +132,12 @@ def drop_indices(): def create_indices(): """Create indices for AircraftBeacon.""" session.execute(""" - CREATE INDEX idx_aircraft_beacon_location ON aircraft_beacons USING GIST(location); - CREATE INDEX ix_aircraft_beacon_receiver_id ON aircraft_beacons USING BTREE(receiver_id); - CREATE INDEX ix_aircraft_beacon_device_id ON aircraft_beacons USING BTREE(device_id); - CREATE INDEX ix_aircraft_beacon_timestamp ON aircraft_beacons USING BTREE(timestamp); + CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name); + CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address); + CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp); + CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location); + + CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance) """) print("Created indices for AircraftBeacon") diff --git a/ogn/model/aircraft_beacon.py b/ogn/model/aircraft_beacon.py index f9439c1..a00c410 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn/model/aircraft_beacon.py @@ -1,6 +1,6 @@ from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger, ForeignKey, Index from sqlalchemy.orm import relationship - +from sqlalchemy.sql import func from .beacon import Beacon @@ -24,6 +24,7 @@ class AircraftBeacon(Beacon): real_address = Column(String(6)) signal_power = Column(Float(precision=2)) + # Not so very important data proximity = None gps_satellites = None gps_quality = None @@ -51,6 +52,9 @@ class AircraftBeacon(Beacon): # Multi-column indices Index('ix_aircraft_beacons_receiver_id_receiver_name', 'receiver_id', 'receiver_name') Index('ix_aircraft_beacons_device_id_address', 'device_id', 'address') + Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp') + + #Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(self.timestamp), 'receiver_id', 'distance') def __repr__(self): return "" % ( diff --git a/ogn/model/airport.py b/ogn/model/airport.py index 0162edc..85b1b79 100644 --- a/ogn/model/airport.py +++ b/ogn/model/airport.py @@ -11,16 +11,16 @@ class Airport(Base): id = Column(Integer, primary_key=True) location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Integer) + altitude = Column(Float(precision=2)) name = Column(String, index=True) code = Column(String(6)) country_code = Column(String(2)) style = Column(SmallInteger) description = Column(String) - runway_direction = Column(Integer) - runway_length = Column(Integer) - frequency = Column(Float) + runway_direction = Column(SmallInteger) + runway_length = Column(SmallInteger) + frequency = Column(Float(precision=2)) def __repr__(self): return "" % ( diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index 56a0326..32831f9 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -1,6 +1,6 @@ from geoalchemy2.shape import to_shape from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, Float, DateTime +from sqlalchemy import Column, String, Integer, SmallInteger, Float, DateTime from sqlalchemy.ext.declarative import AbstractConcreteBase from .base import Base @@ -20,7 +20,7 @@ class Beacon(AbstractConcreteBase, Base): timestamp = Column(DateTime, index=True) symboltable = None symbolcode = None - track = Column(Integer) + track = Column(SmallInteger) ground_speed = Column(Float(precision=2)) comment = None diff --git a/ogn/model/device.py b/ogn/model/device.py index d33b9b3..9529eb6 100644 --- a/ogn/model/device.py +++ b/ogn/model/device.py @@ -14,7 +14,7 @@ class Device(Base): lastseen = Column(DateTime, index=True) aircraft_type = Column(SmallInteger, index=True) stealth = Column(Boolean) - software_version = Column(Float) + software_version = Column(Float(precision=2)) hardware_version = Column(SmallInteger) real_address = Column(String(6)) diff --git a/ogn/model/device_stats.py b/ogn/model/device_stats.py index af3d8eb..c304738 100644 --- a/ogn/model/device_stats.py +++ b/ogn/model/device_stats.py @@ -10,17 +10,21 @@ class DeviceStats(Base): id = Column(Integer, primary_key=True) date = Column(Date) - max_altitude = Column(Float) - receiver_count = Column(Integer) + + # Statistic data + max_altitude = Column(Float(precision=2)) + receiver_count = Column(SmallInteger) aircraft_beacon_count = Column(Integer) firstseen = Column(DateTime) lastseen = Column(DateTime) aircraft_type = Column(SmallInteger) stealth = Column(Boolean) - software_version = Column(Float) + software_version = Column(Float(precision=2)) hardware_version = Column(SmallInteger) real_address = Column(String(6)) + ambiguous = Column(Boolean) + # Relations device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) device = relationship('Device', foreign_keys=[device_id], backref='stats') diff --git a/ogn/model/logbook.py b/ogn/model/logbook.py index 87ad1b8..b21e95c 100644 --- a/ogn/model/logbook.py +++ b/ogn/model/logbook.py @@ -1,4 +1,4 @@ -from sqlalchemy import Integer, DateTime, Column, ForeignKey, case, null +from sqlalchemy import Integer, SmallInteger, Float, DateTime, Column, ForeignKey, case, null from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship @@ -12,10 +12,10 @@ class Logbook(Base): reftime = Column(DateTime, index=True) takeoff_timestamp = Column(DateTime) - takeoff_track = Column(Integer) + takeoff_track = Column(SmallInteger) landing_timestamp = Column(DateTime) - landing_track = Column(Integer) - max_altitude = Column(Integer) + landing_track = Column(SmallInteger) + max_altitude = Column(Float(precision=2)) # Relations takeoff_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True) diff --git a/ogn/model/receiver.py b/ogn/model/receiver.py index bb7453a..3f86a88 100644 --- a/ogn/model/receiver.py +++ b/ogn/model/receiver.py @@ -1,6 +1,6 @@ from geoalchemy2.shape import to_shape from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, DateTime +from sqlalchemy import Column, Float, String, Integer, DateTime from sqlalchemy.orm import relationship from .base import Base @@ -13,7 +13,7 @@ class Receiver(Base): id = Column(Integer, primary_key=True) location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Integer) + altitude = Column(Float(precision=2)) name = Column(String(9)) firstseen = Column(DateTime, index=True) diff --git a/ogn/model/receiver_coverage.py b/ogn/model/receiver_coverage.py index 2cd06ab..1c5dc9a 100644 --- a/ogn/model/receiver_coverage.py +++ b/ogn/model/receiver_coverage.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, String, Integer, Float, Date, ForeignKey, Index +from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index from sqlalchemy.orm import relationship @@ -13,11 +13,11 @@ class ReceiverCoverage(Base): date = Column(Date, primary_key=True) max_signal_quality = Column(Float) - max_altitude = Column(Integer) - min_altitude = Column(Integer) + max_altitude = Column(Float(precision=2)) + min_altitude = Column(Float(precision=2)) aircraft_beacon_count = Column(Integer) - device_count = Column(Integer) + device_count = Column(SmallInteger) # Relations receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_coverages') diff --git a/ogn/model/receiver_stats.py b/ogn/model/receiver_stats.py index 2645779..5f7a8d7 100644 --- a/ogn/model/receiver_stats.py +++ b/ogn/model/receiver_stats.py @@ -1,5 +1,6 @@ -from sqlalchemy import Column, Integer, Date, Float, ForeignKey +from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String from sqlalchemy.orm import relationship +from geoalchemy2.types import Geometry from .base import Base @@ -10,10 +11,17 @@ class ReceiverStats(Base): id = Column(Integer, primary_key=True) date = Column(Date) + + # Statistic data aircraft_beacon_count = Column(Integer) - receiver_beacon_count = Column(Integer) - aircraft_count = Column(Integer) + aircraft_count = Column(SmallInteger) max_distance = Column(Float) + firstseen = Column(DateTime, index=True) + lastseen = Column(DateTime, index=True) + location_wkt = Column('location', Geometry('POINT', srid=4326)) + altitude = Column(Float(precision=2)) + version = Column(String) + platform = Column(String) # Relations receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True) diff --git a/ogn/model/takeoff_landing.py b/ogn/model/takeoff_landing.py index 621e491..c66ca45 100644 --- a/ogn/model/takeoff_landing.py +++ b/ogn/model/takeoff_landing.py @@ -1,4 +1,4 @@ -from sqlalchemy import Boolean, Column, Integer, DateTime, ForeignKey +from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey from sqlalchemy.orm import relationship from .base import Base @@ -12,7 +12,7 @@ class TakeoffLanding(Base): timestamp = Column(DateTime, primary_key=True) is_takeoff = Column(Boolean) - track = Column(Integer) + track = Column(SmallInteger) # Relations airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings')