Use better data types

ogn/collect/stats.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from celery.utils.log import get_task_logger
 
 from sqlalchemy import insert, distinct
-from sqlalchemy.sql import null, and_, func, or_
+from sqlalchemy.sql import null, and_, func, or_, update
 from sqlalchemy.sql.expression import literal_column, case
 
 from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
@@ -35,7 +35,7 @@ def update_device_stats(session=None, date=None):
             func.dense_rank()
                 .over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id)
                 .label('dr')) \
-        .filter(and_(AircraftBeacon.device_id != null(), func.date(AircraftBeacon.timestamp) == date)) \
+        .filter(and_(func.date(AircraftBeacon.timestamp) == date, AircraftBeacon.device_id != null())) \
         .filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) \
         .subquery()
 
@@ -89,9 +89,12 @@ def update_device_stats(session=None, date=None):
 
 
 @app.task
-def update_receiver_stats(date=None):
+def update_receiver_stats(session=None, date=None):
     """Add/update receiver stats."""
 
+    if session is None:
+        session = app.session
+
     if not date:
         logger.warn("A date is needed for calculating stats. Exiting")
         return None
@@ -101,25 +104,49 @@ def update_receiver_stats(date=None):
         .filter(ReceiverStats.date == date) \
         .delete()
 
-    # Calculate stats for the selected date
+    # Calculate stats, firstseen, lastseen and last values != NULL
     receiver_stats = session.query(
-        AircraftBeacon.receiver_id,
+            distinct(ReceiverBeacon.receiver_id).label('receiver_id'),
-        literal_column("'{}'".format(date)).label('date'),
+            func.date(ReceiverBeacon.timestamp).label('date'),
-        func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
+            func.first_value(ReceiverBeacon.timestamp)
-        func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'),
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast())
-        func.max(AircraftBeacon.distance).label('max_distance')) \
+                .label('firstseen'),
-        .filter(AircraftBeacon.receiver_id != null()) \
+            func.first_value(ReceiverBeacon.timestamp)
-        .filter(func.date(AircraftBeacon.timestamp) == date) \
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
-        .group_by(AircraftBeacon.receiver_id) \
+                .label('lastseen'),
+            func.first_value(ReceiverBeacon.location_wkt)
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
+                .label('location_wkt'),
+            func.first_value(ReceiverBeacon.altitude)
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
+                .label('altitude'),
+            func.first_value(ReceiverBeacon.version)
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
+                .label('version'),
+            func.first_value(ReceiverBeacon.platform)
+                .over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
+                .label('platform')) \
         .subquery()
 
     # And insert them
     ins = insert(ReceiverStats).from_select(
-        [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance],
+        [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.firstseen, ReceiverStats.lastseen, ReceiverStats.location_wkt, ReceiverStats.altitude, ReceiverStats.version, ReceiverStats.platform],
         receiver_stats)
     res = session.execute(ins)
     insert_counter = res.rowcount
     session.commit()
-    logger.debug("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
+    logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
 
-    return "ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
+    # Update AircraftBeacon distances
+    upd = update(AircraftBeacon) \
+        .where(and_(func.date(AircraftBeacon.timestamp) == ReceiverStats.date,
+                    AircraftBeacon.receiver_id == ReceiverStats.receiver_id,
+                    AircraftBeacon.distance == null())) \
+        .values({"distance": func.ST_Distance_Sphere(AircraftBeacon.location_wkt, ReceiverStats.location_wkt)})
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.warn("Updated {} AircraftBeacons".format(update_counter))
+
+    return "ReceiverStats for {}: {} deleted, {} inserted, AircraftBeacons: {} updated".format(date, deleted_counter, insert_counter, update_counter)

ogn/collect/takeoff_landings.py

@@ -1,8 +1,8 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from celery.utils.log import get_task_logger
 
-from sqlalchemy import and_, or_, insert, between, exists
+from sqlalchemy import and_, or_, insert, update, between, exists
 from sqlalchemy.sql import func, null
 from sqlalchemy.sql.expression import case
 
@@ -13,7 +13,7 @@ logger = get_task_logger(__name__)
 
 
 @app.task
-def update_takeoff_landings(session=None):
+def update_takeoff_landings(session=None, date=None):
     """Compute takeoffs and landings."""
 
     logger.info("Compute takeoffs and landings.")
@@ -43,10 +43,20 @@ def update_takeoff_landings(session=None):
               AircraftBeacon.receiver_id)
 
     # make a query with current, previous and next position
-    beacon_selection = session.query(AircraftBeacon.id) \
+    if date is None:
-        .order_by(AircraftBeacon.timestamp) \
+        beacon_selection = session.query(AircraftBeacon.id) \
-        .limit(1000000) \
+            .filter(AircraftBeacon.status == 0) \
-        .subquery()
+            .order_by(AircraftBeacon.timestamp) \
+            .subquery()
+    else:
+        my_day = datetime.strptime(date, '%Y-%m-%d')
+        beacon_selection = session.query(AircraftBeacon.id) \
+            .filter(and_(AircraftBeacon.status == 0,
+                         AircraftBeacon.timestamp >= my_day - timedelta(minutes=5),
+                         AircraftBeacon.timestamp < my_day + timedelta(days=1, minutes=5))) \
+            .order_by(AircraftBeacon.timestamp) \
+            .limit(100000) \
+            .subquery()
 
     sq = session.query(
         AircraftBeacon.id,
@@ -78,6 +88,9 @@ def update_takeoff_landings(session=None):
        .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
        .subquery()
 
+    logger.warn(sq2)
+    return
+
     # find possible takeoffs and landings
     sq3 = session.query(
         sq2.c.id,
@@ -129,9 +142,17 @@ def update_takeoff_landings(session=None):
                                               TakeoffLanding.airport_id),
                                              takeoff_landing_query)
     result = session.execute(ins)
-    counter = result.rowcount
+    insert_counter = result.rowcount
+    logger.warn("Inserted {} TakeoffLandings".format(insert_counter))
 
+    # Set calculated beacons as 'used'
+    upd = update(AircraftBeacon) \
+        .where(AircraftBeacon.id == sq2.c.id) \
+        .values({"status": 1})
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
     session.commit()
-    logger.debug("Inserted {} TakeoffLandings".format(counter))
+    logger.warn("Updated {} AircraftBeacons".format(update_counter))
 
-    return "Inserted {} TakeoffLandings".format(counter)
+    return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(insert_counter, update_counter)

ogn/commands/bulkimport.py

@@ -112,9 +112,11 @@ def convert(sourcefile, path=''):
 def drop_indices():
     """Drop indices of AircraftBeacon."""
     session.execute("""
-        DROP INDEX IF EXISTS idx_aircraft_beacons_location;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address;
-        DROP INDEX IF EXISTS ix_aircraft_beacons_device_id;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_location;
+        DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs;
         DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp;
     """)
     print("Dropped indices of AircraftBeacon")
@@ -130,10 +132,12 @@ def drop_indices():
 def create_indices():
     """Create indices for AircraftBeacon."""
     session.execute("""
-        CREATE INDEX idx_aircraft_beacon_location ON aircraft_beacons USING GIST(location);
+        CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name);
-        CREATE INDEX ix_aircraft_beacon_receiver_id ON aircraft_beacons USING BTREE(receiver_id);
+        CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address);
-        CREATE INDEX ix_aircraft_beacon_device_id ON aircraft_beacons USING BTREE(device_id);
+        CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp);
-        CREATE INDEX ix_aircraft_beacon_timestamp ON aircraft_beacons USING BTREE(timestamp);
+        CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
+
+        CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance)
     """)
     print("Created indices for AircraftBeacon")
 

ogn/model/aircraft_beacon.py

@@ -1,6 +1,6 @@
 from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger, ForeignKey, Index
 from sqlalchemy.orm import relationship
-
+from sqlalchemy.sql import func
 from .beacon import Beacon
 
 
@@ -24,6 +24,7 @@ class AircraftBeacon(Beacon):
     real_address = Column(String(6))
     signal_power = Column(Float(precision=2))
 
+    # Not so very important data
     proximity = None
     gps_satellites = None
     gps_quality = None
@@ -51,6 +52,9 @@ class AircraftBeacon(Beacon):
     # Multi-column indices
     Index('ix_aircraft_beacons_receiver_id_receiver_name', 'receiver_id', 'receiver_name')
     Index('ix_aircraft_beacons_device_id_address', 'device_id', 'address')
+    Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
+
+    #Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(self.timestamp), 'receiver_id', 'distance')
 
     def __repr__(self):
         return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (

ogn/model/airport.py

@@ -11,16 +11,16 @@ class Airport(Base):
     id = Column(Integer, primary_key=True)
 
     location_wkt = Column('location', Geometry('POINT', srid=4326))
-    altitude = Column(Integer)
+    altitude = Column(Float(precision=2))
 
     name = Column(String, index=True)
     code = Column(String(6))
     country_code = Column(String(2))
     style = Column(SmallInteger)
     description = Column(String)
-    runway_direction = Column(Integer)
+    runway_direction = Column(SmallInteger)
-    runway_length = Column(Integer)
+    runway_length = Column(SmallInteger)
-    frequency = Column(Float)
+    frequency = Column(Float(precision=2))
 
     def __repr__(self):
         return "<Airport %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,% s>" % (

ogn/model/beacon.py

@@ -1,6 +1,6 @@
 from geoalchemy2.shape import to_shape
 from geoalchemy2.types import Geometry
-from sqlalchemy import Column, String, Integer, Float, DateTime
+from sqlalchemy import Column, String, Integer, SmallInteger, Float, DateTime
 from sqlalchemy.ext.declarative import AbstractConcreteBase
 
 from .base import Base
@@ -20,7 +20,7 @@ class Beacon(AbstractConcreteBase, Base):
     timestamp = Column(DateTime, index=True)
     symboltable = None
     symbolcode = None
-    track = Column(Integer)
+    track = Column(SmallInteger)
     ground_speed = Column(Float(precision=2))
     comment = None
 

ogn/model/device.py

@@ -14,7 +14,7 @@ class Device(Base):
     lastseen = Column(DateTime, index=True)
     aircraft_type = Column(SmallInteger, index=True)
     stealth = Column(Boolean)
-    software_version = Column(Float)
+    software_version = Column(Float(precision=2))
     hardware_version = Column(SmallInteger)
     real_address = Column(String(6))
 

ogn/model/device_stats.py

@@ -10,17 +10,21 @@ class DeviceStats(Base):
     id = Column(Integer, primary_key=True)
 
     date = Column(Date)
-    max_altitude = Column(Float)
+
-    receiver_count = Column(Integer)
+    # Statistic data
+    max_altitude = Column(Float(precision=2))
+    receiver_count = Column(SmallInteger)
     aircraft_beacon_count = Column(Integer)
     firstseen = Column(DateTime)
     lastseen = Column(DateTime)
     aircraft_type = Column(SmallInteger)
     stealth = Column(Boolean)
-    software_version = Column(Float)
+    software_version = Column(Float(precision=2))
     hardware_version = Column(SmallInteger)
     real_address = Column(String(6))
 
+    ambiguous = Column(Boolean)
+
     # Relations
     device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
     device = relationship('Device', foreign_keys=[device_id], backref='stats')

ogn/model/logbook.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Integer, DateTime, Column, ForeignKey, case, null
+from sqlalchemy import Integer, SmallInteger, Float, DateTime, Column, ForeignKey, case, null
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 
@@ -12,10 +12,10 @@ class Logbook(Base):
 
     reftime = Column(DateTime, index=True)
     takeoff_timestamp = Column(DateTime)
-    takeoff_track = Column(Integer)
+    takeoff_track = Column(SmallInteger)
     landing_timestamp = Column(DateTime)
-    landing_track = Column(Integer)
+    landing_track = Column(SmallInteger)
-    max_altitude = Column(Integer)
+    max_altitude = Column(Float(precision=2))
 
     # Relations
     takeoff_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True)

ogn/model/receiver.py

@@ -1,6 +1,6 @@
 from geoalchemy2.shape import to_shape
 from geoalchemy2.types import Geometry
-from sqlalchemy import Column, String, Integer, DateTime
+from sqlalchemy import Column, Float, String, Integer, DateTime
 from sqlalchemy.orm import relationship
 
 from .base import Base
@@ -13,7 +13,7 @@ class Receiver(Base):
     id = Column(Integer, primary_key=True)
 
     location_wkt = Column('location', Geometry('POINT', srid=4326))
-    altitude = Column(Integer)
+    altitude = Column(Float(precision=2))
 
     name = Column(String(9))
     firstseen = Column(DateTime, index=True)

ogn/model/receiver_coverage.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, String, Integer, Float, Date, ForeignKey, Index
+from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index
 from sqlalchemy.orm import relationship
 
 
@@ -13,11 +13,11 @@ class ReceiverCoverage(Base):
     date = Column(Date, primary_key=True)
 
     max_signal_quality = Column(Float)
-    max_altitude = Column(Integer)
+    max_altitude = Column(Float(precision=2))
-    min_altitude = Column(Integer)
+    min_altitude = Column(Float(precision=2))
     aircraft_beacon_count = Column(Integer)
 
-    device_count = Column(Integer)
+    device_count = Column(SmallInteger)
 
     # Relations
     receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_coverages')

ogn/model/receiver_stats.py

@@ -1,5 +1,6 @@
-from sqlalchemy import Column, Integer, Date, Float, ForeignKey
+from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String
 from sqlalchemy.orm import relationship
+from geoalchemy2.types import Geometry
 
 from .base import Base
 
@@ -10,10 +11,17 @@ class ReceiverStats(Base):
     id = Column(Integer, primary_key=True)
 
     date = Column(Date)
+
+    # Statistic data
     aircraft_beacon_count = Column(Integer)
-    receiver_beacon_count = Column(Integer)
+    aircraft_count = Column(SmallInteger)
-    aircraft_count = Column(Integer)
     max_distance = Column(Float)
+    firstseen = Column(DateTime, index=True)
+    lastseen = Column(DateTime, index=True)
+    location_wkt = Column('location', Geometry('POINT', srid=4326))
+    altitude = Column(Float(precision=2))
+    version = Column(String)
+    platform = Column(String)
 
     # Relations
     receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)

ogn/model/takeoff_landing.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Boolean, Column, Integer, DateTime, ForeignKey
+from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey
 from sqlalchemy.orm import relationship
 
 from .base import Base
@@ -12,7 +12,7 @@ class TakeoffLanding(Base):
     timestamp = Column(DateTime, primary_key=True)
 
     is_takeoff = Column(Boolean)
-    track = Column(Integer)
+    track = Column(SmallInteger)
 
     # Relations
     airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings')