kopia lustrzana https://github.com/glidernet/ogn-python
Use better data types
Konstantin Gründger
rodzic
36239be842
commit
8159646fc2
|
|
@ -3,7 +3,7 @@ from datetime import datetime
|
|||
from celery.utils.log import get_task_logger
|
||||
|
||||
from sqlalchemy import insert, distinct
|
||||
from sqlalchemy.sql import null, and_, func, or_
|
||||
from sqlalchemy.sql import null, and_, func, or_, update
|
||||
from sqlalchemy.sql.expression import literal_column, case
|
||||
|
||||
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
|
||||
|
|
@ -35,7 +35,7 @@ def update_device_stats(session=None, date=None):
|
|||
func.dense_rank()
|
||||
.over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id)
|
||||
.label('dr')) \
|
||||
.filter(and_(AircraftBeacon.device_id != null(), func.date(AircraftBeacon.timestamp) == date)) \
|
||||
.filter(and_(func.date(AircraftBeacon.timestamp) == date, AircraftBeacon.device_id != null())) \
|
||||
.filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) \
|
||||
.subquery()
|
||||
|
||||
|
|
@ -89,9 +89,12 @@ def update_device_stats(session=None, date=None):
|
|||
|
||||
|
||||
@app.task
|
||||
def update_receiver_stats(date=None):
|
||||
def update_receiver_stats(session=None, date=None):
|
||||
"""Add/update receiver stats."""
|
||||
|
||||
if session is None:
|
||||
session = app.session
|
||||
|
||||
if not date:
|
||||
logger.warn("A date is needed for calculating stats. Exiting")
|
||||
return None
|
||||
|
|
@ -101,25 +104,49 @@ def update_receiver_stats(date=None):
|
|||
.filter(ReceiverStats.date == date) \
|
||||
.delete()
|
||||
|
||||
# Calculate stats for the selected date
|
||||
# Calculate stats, firstseen, lastseen and last values != NULL
|
||||
receiver_stats = session.query(
|
||||
AircraftBeacon.receiver_id,
|
||||
literal_column("'{}'".format(date)).label('date'),
|
||||
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
|
||||
func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'),
|
||||
func.max(AircraftBeacon.distance).label('max_distance')) \
|
||||
.filter(AircraftBeacon.receiver_id != null()) \
|
||||
.filter(func.date(AircraftBeacon.timestamp) == date) \
|
||||
.group_by(AircraftBeacon.receiver_id) \
|
||||
distinct(ReceiverBeacon.receiver_id).label('receiver_id'),
|
||||
func.date(ReceiverBeacon.timestamp).label('date'),
|
||||
func.first_value(ReceiverBeacon.timestamp)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast())
|
||||
.label('firstseen'),
|
||||
func.first_value(ReceiverBeacon.timestamp)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
|
||||
.label('lastseen'),
|
||||
func.first_value(ReceiverBeacon.location_wkt)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
|
||||
.label('location_wkt'),
|
||||
func.first_value(ReceiverBeacon.altitude)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
|
||||
.label('altitude'),
|
||||
func.first_value(ReceiverBeacon.version)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
|
||||
.label('version'),
|
||||
func.first_value(ReceiverBeacon.platform)
|
||||
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
|
||||
.label('platform')) \
|
||||
.subquery()
|
||||
|
||||
# And insert them
|
||||
ins = insert(ReceiverStats).from_select(
|
||||
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance],
|
||||
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.firstseen, ReceiverStats.lastseen, ReceiverStats.location_wkt, ReceiverStats.altitude, ReceiverStats.version, ReceiverStats.platform],
|
||||
receiver_stats)
|
||||
res = session.execute(ins)
|
||||
insert_counter = res.rowcount
|
||||
session.commit()
|
||||
logger.debug("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
|
||||
logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
|
||||
|
||||
return "ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
|
||||
# Update AircraftBeacon distances
|
||||
upd = update(AircraftBeacon) \
|
||||
.where(and_(func.date(AircraftBeacon.timestamp) == ReceiverStats.date,
|
||||
AircraftBeacon.receiver_id == ReceiverStats.receiver_id,
|
||||
AircraftBeacon.distance == null())) \
|
||||
.values({"distance": func.ST_Distance_Sphere(AircraftBeacon.location_wkt, ReceiverStats.location_wkt)})
|
||||
|
||||
result = session.execute(upd)
|
||||
update_counter = result.rowcount
|
||||
session.commit()
|
||||
logger.warn("Updated {} AircraftBeacons".format(update_counter))
|
||||
|
||||
return "ReceiverStats for {}: {} deleted, {} inserted, AircraftBeacons: {} updated".format(date, deleted_counter, insert_counter, update_counter)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from sqlalchemy import and_, or_, insert, between, exists
|
||||
from sqlalchemy import and_, or_, insert, update, between, exists
|
||||
from sqlalchemy.sql import func, null
|
||||
from sqlalchemy.sql.expression import case
|
||||
|
||||
|
|
@ -13,7 +13,7 @@ logger = get_task_logger(__name__)
|
|||
|
||||
|
||||
@app.task
|
||||
def update_takeoff_landings(session=None):
|
||||
def update_takeoff_landings(session=None, date=None):
|
||||
"""Compute takeoffs and landings."""
|
||||
|
||||
logger.info("Compute takeoffs and landings.")
|
||||
|
|
@ -43,10 +43,20 @@ def update_takeoff_landings(session=None):
|
|||
AircraftBeacon.receiver_id)
|
||||
|
||||
# make a query with current, previous and next position
|
||||
beacon_selection = session.query(AircraftBeacon.id) \
|
||||
.order_by(AircraftBeacon.timestamp) \
|
||||
.limit(1000000) \
|
||||
.subquery()
|
||||
if date is None:
|
||||
beacon_selection = session.query(AircraftBeacon.id) \
|
||||
.filter(AircraftBeacon.status == 0) \
|
||||
.order_by(AircraftBeacon.timestamp) \
|
||||
.subquery()
|
||||
else:
|
||||
my_day = datetime.strptime(date, '%Y-%m-%d')
|
||||
beacon_selection = session.query(AircraftBeacon.id) \
|
||||
.filter(and_(AircraftBeacon.status == 0,
|
||||
AircraftBeacon.timestamp >= my_day - timedelta(minutes=5),
|
||||
AircraftBeacon.timestamp < my_day + timedelta(days=1, minutes=5))) \
|
||||
.order_by(AircraftBeacon.timestamp) \
|
||||
.limit(100000) \
|
||||
.subquery()
|
||||
|
||||
sq = session.query(
|
||||
AircraftBeacon.id,
|
||||
|
|
@ -78,6 +88,9 @@ def update_takeoff_landings(session=None):
|
|||
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
|
||||
.subquery()
|
||||
|
||||
logger.warn(sq2)
|
||||
return
|
||||
|
||||
# find possible takeoffs and landings
|
||||
sq3 = session.query(
|
||||
sq2.c.id,
|
||||
|
|
@ -129,9 +142,17 @@ def update_takeoff_landings(session=None):
|
|||
TakeoffLanding.airport_id),
|
||||
takeoff_landing_query)
|
||||
result = session.execute(ins)
|
||||
counter = result.rowcount
|
||||
insert_counter = result.rowcount
|
||||
logger.warn("Inserted {} TakeoffLandings".format(insert_counter))
|
||||
|
||||
# Set calculated beacons as 'used'
|
||||
upd = update(AircraftBeacon) \
|
||||
.where(AircraftBeacon.id == sq2.c.id) \
|
||||
.values({"status": 1})
|
||||
|
||||
result = session.execute(upd)
|
||||
update_counter = result.rowcount
|
||||
session.commit()
|
||||
logger.debug("Inserted {} TakeoffLandings".format(counter))
|
||||
logger.warn("Updated {} AircraftBeacons".format(update_counter))
|
||||
|
||||
return "Inserted {} TakeoffLandings".format(counter)
|
||||
return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(insert_counter, update_counter)
|
||||
|
|
|
|||
|
|
@ -112,9 +112,11 @@ def convert(sourcefile, path=''):
|
|||
def drop_indices():
|
||||
"""Drop indices of AircraftBeacon."""
|
||||
session.execute("""
|
||||
DROP INDEX IF EXISTS idx_aircraft_beacons_location;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_location;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs;
|
||||
DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp;
|
||||
""")
|
||||
print("Dropped indices of AircraftBeacon")
|
||||
|
|
@ -130,10 +132,12 @@ def drop_indices():
|
|||
def create_indices():
|
||||
"""Create indices for AircraftBeacon."""
|
||||
session.execute("""
|
||||
CREATE INDEX idx_aircraft_beacon_location ON aircraft_beacons USING GIST(location);
|
||||
CREATE INDEX ix_aircraft_beacon_receiver_id ON aircraft_beacons USING BTREE(receiver_id);
|
||||
CREATE INDEX ix_aircraft_beacon_device_id ON aircraft_beacons USING BTREE(device_id);
|
||||
CREATE INDEX ix_aircraft_beacon_timestamp ON aircraft_beacons USING BTREE(timestamp);
|
||||
CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name);
|
||||
CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address);
|
||||
CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp);
|
||||
CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
|
||||
|
||||
CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance)
|
||||
""")
|
||||
print("Created indices for AircraftBeacon")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger, ForeignKey, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from sqlalchemy.sql import func
|
||||
from .beacon import Beacon
|
||||
|
||||
|
||||
|
|
@ -24,6 +24,7 @@ class AircraftBeacon(Beacon):
|
|||
real_address = Column(String(6))
|
||||
signal_power = Column(Float(precision=2))
|
||||
|
||||
# Not so very important data
|
||||
proximity = None
|
||||
gps_satellites = None
|
||||
gps_quality = None
|
||||
|
|
@ -51,6 +52,9 @@ class AircraftBeacon(Beacon):
|
|||
# Multi-column indices
|
||||
Index('ix_aircraft_beacons_receiver_id_receiver_name', 'receiver_id', 'receiver_name')
|
||||
Index('ix_aircraft_beacons_device_id_address', 'device_id', 'address')
|
||||
Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
|
||||
|
||||
#Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(self.timestamp), 'receiver_id', 'distance')
|
||||
|
||||
def __repr__(self):
|
||||
return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
|
|
|
|||
|
|
@ -11,16 +11,16 @@ class Airport(Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Integer)
|
||||
altitude = Column(Float(precision=2))
|
||||
|
||||
name = Column(String, index=True)
|
||||
code = Column(String(6))
|
||||
country_code = Column(String(2))
|
||||
style = Column(SmallInteger)
|
||||
description = Column(String)
|
||||
runway_direction = Column(Integer)
|
||||
runway_length = Column(Integer)
|
||||
frequency = Column(Float)
|
||||
runway_direction = Column(SmallInteger)
|
||||
runway_length = Column(SmallInteger)
|
||||
frequency = Column(Float(precision=2))
|
||||
|
||||
def __repr__(self):
|
||||
return "<Airport %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,% s>" % (
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from geoalchemy2.shape import to_shape
|
||||
from geoalchemy2.types import Geometry
|
||||
from sqlalchemy import Column, String, Integer, Float, DateTime
|
||||
from sqlalchemy import Column, String, Integer, SmallInteger, Float, DateTime
|
||||
from sqlalchemy.ext.declarative import AbstractConcreteBase
|
||||
|
||||
from .base import Base
|
||||
|
|
@ -20,7 +20,7 @@ class Beacon(AbstractConcreteBase, Base):
|
|||
timestamp = Column(DateTime, index=True)
|
||||
symboltable = None
|
||||
symbolcode = None
|
||||
track = Column(Integer)
|
||||
track = Column(SmallInteger)
|
||||
ground_speed = Column(Float(precision=2))
|
||||
comment = None
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class Device(Base):
|
|||
lastseen = Column(DateTime, index=True)
|
||||
aircraft_type = Column(SmallInteger, index=True)
|
||||
stealth = Column(Boolean)
|
||||
software_version = Column(Float)
|
||||
software_version = Column(Float(precision=2))
|
||||
hardware_version = Column(SmallInteger)
|
||||
real_address = Column(String(6))
|
||||
|
||||
|
|
|
|||
|
|
@ -10,17 +10,21 @@ class DeviceStats(Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
|
||||
date = Column(Date)
|
||||
max_altitude = Column(Float)
|
||||
receiver_count = Column(Integer)
|
||||
|
||||
# Statistic data
|
||||
max_altitude = Column(Float(precision=2))
|
||||
receiver_count = Column(SmallInteger)
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
firstseen = Column(DateTime)
|
||||
lastseen = Column(DateTime)
|
||||
aircraft_type = Column(SmallInteger)
|
||||
stealth = Column(Boolean)
|
||||
software_version = Column(Float)
|
||||
software_version = Column(Float(precision=2))
|
||||
hardware_version = Column(SmallInteger)
|
||||
real_address = Column(String(6))
|
||||
|
||||
ambiguous = Column(Boolean)
|
||||
|
||||
# Relations
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
|
||||
device = relationship('Device', foreign_keys=[device_id], backref='stats')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from sqlalchemy import Integer, DateTime, Column, ForeignKey, case, null
|
||||
from sqlalchemy import Integer, SmallInteger, Float, DateTime, Column, ForeignKey, case, null
|
||||
from sqlalchemy.ext.hybrid import hybrid_property
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
|
|
@ -12,10 +12,10 @@ class Logbook(Base):
|
|||
|
||||
reftime = Column(DateTime, index=True)
|
||||
takeoff_timestamp = Column(DateTime)
|
||||
takeoff_track = Column(Integer)
|
||||
takeoff_track = Column(SmallInteger)
|
||||
landing_timestamp = Column(DateTime)
|
||||
landing_track = Column(Integer)
|
||||
max_altitude = Column(Integer)
|
||||
landing_track = Column(SmallInteger)
|
||||
max_altitude = Column(Float(precision=2))
|
||||
|
||||
# Relations
|
||||
takeoff_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from geoalchemy2.shape import to_shape
|
||||
from geoalchemy2.types import Geometry
|
||||
from sqlalchemy import Column, String, Integer, DateTime
|
||||
from sqlalchemy import Column, Float, String, Integer, DateTime
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from .base import Base
|
||||
|
|
@ -13,7 +13,7 @@ class Receiver(Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Integer)
|
||||
altitude = Column(Float(precision=2))
|
||||
|
||||
name = Column(String(9))
|
||||
firstseen = Column(DateTime, index=True)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from sqlalchemy import Column, String, Integer, Float, Date, ForeignKey, Index
|
||||
from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
|
||||
|
|
@ -13,11 +13,11 @@ class ReceiverCoverage(Base):
|
|||
date = Column(Date, primary_key=True)
|
||||
|
||||
max_signal_quality = Column(Float)
|
||||
max_altitude = Column(Integer)
|
||||
min_altitude = Column(Integer)
|
||||
max_altitude = Column(Float(precision=2))
|
||||
min_altitude = Column(Float(precision=2))
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
|
||||
device_count = Column(Integer)
|
||||
device_count = Column(SmallInteger)
|
||||
|
||||
# Relations
|
||||
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_coverages')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
|
||||
from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String
|
||||
from sqlalchemy.orm import relationship
|
||||
from geoalchemy2.types import Geometry
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
|
@ -10,10 +11,17 @@ class ReceiverStats(Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
|
||||
date = Column(Date)
|
||||
|
||||
# Statistic data
|
||||
aircraft_beacon_count = Column(Integer)
|
||||
receiver_beacon_count = Column(Integer)
|
||||
aircraft_count = Column(Integer)
|
||||
aircraft_count = Column(SmallInteger)
|
||||
max_distance = Column(Float)
|
||||
firstseen = Column(DateTime, index=True)
|
||||
lastseen = Column(DateTime, index=True)
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Float(precision=2))
|
||||
version = Column(String)
|
||||
platform = Column(String)
|
||||
|
||||
# Relations
|
||||
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from sqlalchemy import Boolean, Column, Integer, DateTime, ForeignKey
|
||||
from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from .base import Base
|
||||
|
|
@ -12,7 +12,7 @@ class TakeoffLanding(Base):
|
|||
timestamp = Column(DateTime, primary_key=True)
|
||||
|
||||
is_takeoff = Column(Boolean)
|
||||
track = Column(Integer)
|
||||
track = Column(SmallInteger)
|
||||
|
||||
# Relations
|
||||
airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings')
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue