mirror
/
ogn-python
kopia lustrzana https://github.com/glidernet/ogn-python
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Use better data types

pull/68/head
Konstantin Gründger 2018-01-21 21:06:27 +01:00
rodzic 36239be842
commit 8159646fc2
13 zmienionych plików z 126 dodań i 58 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

57
ogn/collect/stats.py
Wyświetl plik

@ -3,7 +3,7 @@ from datetime import datetime
from celery.utils.log import get_task_logger
from sqlalchemy import insert, distinct
from sqlalchemy.sql import null, and_, func, or_
from sqlalchemy.sql import null, and_, func, or_, update
from sqlalchemy.sql.expression import literal_column, case
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
@ -35,7 +35,7 @@ def update_device_stats(session=None, date=None):
func.dense_rank()
.over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id)
.label('dr')) \
.filter(and_(AircraftBeacon.device_id != null(), func.date(AircraftBeacon.timestamp) == date)) \
.filter(and_(func.date(AircraftBeacon.timestamp) == date, AircraftBeacon.device_id != null())) \
.filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) \
.subquery()
@ -89,9 +89,12 @@ def update_device_stats(session=None, date=None):
@app.task
def update_receiver_stats(date=None):
def update_receiver_stats(session=None, date=None):
"""Add/update receiver stats."""
if session is None:
session = app.session
if not date:
logger.warn("A date is needed for calculating stats. Exiting")
return None
@ -101,25 +104,49 @@ def update_receiver_stats(date=None):
.filter(ReceiverStats.date == date) \
.delete()
# Calculate stats for the selected date
# Calculate stats, firstseen, lastseen and last values != NULL
receiver_stats = session.query(
AircraftBeacon.receiver_id,
literal_column("'{}'".format(date)).label('date'),
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'),
func.max(AircraftBeacon.distance).label('max_distance')) \
.filter(AircraftBeacon.receiver_id != null()) \
.filter(func.date(AircraftBeacon.timestamp) == date) \
.group_by(AircraftBeacon.receiver_id) \
distinct(ReceiverBeacon.receiver_id).label('receiver_id'),
func.date(ReceiverBeacon.timestamp).label('date'),
func.first_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast())
.label('firstseen'),
func.first_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('lastseen'),
func.first_value(ReceiverBeacon.location_wkt)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('location_wkt'),
func.first_value(ReceiverBeacon.altitude)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('altitude'),
func.first_value(ReceiverBeacon.version)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('version'),
func.first_value(ReceiverBeacon.platform)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('platform')) \
.subquery()
# And insert them
ins = insert(ReceiverStats).from_select(
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance],
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.firstseen, ReceiverStats.lastseen, ReceiverStats.location_wkt, ReceiverStats.altitude, ReceiverStats.version, ReceiverStats.platform],
receiver_stats)
res = session.execute(ins)
insert_counter = res.rowcount
session.commit()
logger.debug("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
return "ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
# Update AircraftBeacon distances
upd = update(AircraftBeacon) \
.where(and_(func.date(AircraftBeacon.timestamp) == ReceiverStats.date,
AircraftBeacon.receiver_id == ReceiverStats.receiver_id,
AircraftBeacon.distance == null())) \
.values({"distance": func.ST_Distance_Sphere(AircraftBeacon.location_wkt, ReceiverStats.location_wkt)})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} AircraftBeacons".format(update_counter))
return "ReceiverStats for {}: {} deleted, {} inserted, AircraftBeacons: {} updated".format(date, deleted_counter, insert_counter, update_counter)

35
ogn/collect/takeoff_landings.py
Wyświetl plik

@ -1,8 +1,8 @@
from datetime import timedelta
from datetime import datetime, timedelta
from celery.utils.log import get_task_logger
from sqlalchemy import and_, or_, insert, between, exists
from sqlalchemy import and_, or_, insert, update, between, exists
from sqlalchemy.sql import func, null
from sqlalchemy.sql.expression import case
@ -13,7 +13,7 @@ logger = get_task_logger(__name__)
@app.task
def update_takeoff_landings(session=None):
def update_takeoff_landings(session=None, date=None):
"""Compute takeoffs and landings."""
logger.info("Compute takeoffs and landings.")
@ -43,9 +43,19 @@ def update_takeoff_landings(session=None):
AircraftBeacon.receiver_id)
# make a query with current, previous and next position
if date is None:
beacon_selection = session.query(AircraftBeacon.id) \
.filter(AircraftBeacon.status == 0) \
.order_by(AircraftBeacon.timestamp) \
.limit(1000000) \
.subquery()
else:
my_day = datetime.strptime(date, '%Y-%m-%d')
beacon_selection = session.query(AircraftBeacon.id) \
.filter(and_(AircraftBeacon.status == 0,
AircraftBeacon.timestamp >= my_day - timedelta(minutes=5),
AircraftBeacon.timestamp < my_day + timedelta(days=1, minutes=5))) \
.order_by(AircraftBeacon.timestamp) \
.limit(100000) \
.subquery()
sq = session.query(
@ -78,6 +88,9 @@ def update_takeoff_landings(session=None):
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
.subquery()
logger.warn(sq2)
return
# find possible takeoffs and landings
sq3 = session.query(
sq2.c.id,
@ -129,9 +142,17 @@ def update_takeoff_landings(session=None):
TakeoffLanding.airport_id),
takeoff_landing_query)
result = session.execute(ins)
counter = result.rowcount
insert_counter = result.rowcount
logger.warn("Inserted {} TakeoffLandings".format(insert_counter))
# Set calculated beacons as 'used'
upd = update(AircraftBeacon) \
.where(AircraftBeacon.id == sq2.c.id) \
.values({"status": 1})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.debug("Inserted {} TakeoffLandings".format(counter))
logger.warn("Updated {} AircraftBeacons".format(update_counter))
return "Inserted {} TakeoffLandings".format(counter)
return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(insert_counter, update_counter)

18
ogn/commands/bulkimport.py
Wyświetl plik

@ -112,9 +112,11 @@ def convert(sourcefile, path=''):
def drop_indices():
"""Drop indices of AircraftBeacon."""
session.execute("""
DROP INDEX IF EXISTS idx_aircraft_beacons_location;
DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id;
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id;
DROP INDEX IF EXISTS ix_aircraft_beacons_receiver_id_receiver_name;
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_address;
DROP INDEX IF EXISTS ix_aircraft_beacons_device_id_timestamp;
DROP INDEX IF EXISTS ix_aircraft_beacons_location;
DROP INDEX IF EXISTS ix_aircraft_beacons_location_mgrs;
DROP INDEX IF EXISTS ix_aircraft_beacons_timestamp;
""")
print("Dropped indices of AircraftBeacon")
@ -130,10 +132,12 @@ def drop_indices():
def create_indices():
"""Create indices for AircraftBeacon."""
session.execute("""
CREATE INDEX idx_aircraft_beacon_location ON aircraft_beacons USING GIST(location);
CREATE INDEX ix_aircraft_beacon_receiver_id ON aircraft_beacons USING BTREE(receiver_id);
CREATE INDEX ix_aircraft_beacon_device_id ON aircraft_beacons USING BTREE(device_id);
CREATE INDEX ix_aircraft_beacon_timestamp ON aircraft_beacons USING BTREE(timestamp);
CREATE INDEX ix_aircraft_beacons_receiver_id_receiver_name ON aircraft_beacons USING BTREE(receiver_id, receiver_name);
CREATE INDEX ix_aircraft_beacons_device_id_address ON aircraft_beacons USING BTREE(device_id, address);
CREATE INDEX ix_aircraft_beacons_device_id_timestamp ON aircraft_beacons USING BTREE(device_id, timestamp);
CREATE INDEX ix_aircraft_beacons_location ON aircraft_beacons USING GIST(location);
CREATE INDEX ix_aircraft_beacons_date_receiver_id_distance ON aircraft_beacons USING btree((timestamp::date), receiver_id, distance)
""")
print("Created indices for AircraftBeacon")

6
ogn/model/aircraft_beacon.py
Wyświetl plik

@ -1,6 +1,6 @@
from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger, ForeignKey, Index
from sqlalchemy.orm import relationship
from sqlalchemy.sql import func
from .beacon import Beacon
@ -24,6 +24,7 @@ class AircraftBeacon(Beacon):
real_address = Column(String(6))
signal_power = Column(Float(precision=2))
# Not so very important data
proximity = None
gps_satellites = None
gps_quality = None
@ -51,6 +52,9 @@ class AircraftBeacon(Beacon):
# Multi-column indices
Index('ix_aircraft_beacons_receiver_id_receiver_name', 'receiver_id', 'receiver_name')
Index('ix_aircraft_beacons_device_id_address', 'device_id', 'address')
Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
#Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(self.timestamp), 'receiver_id', 'distance')
def __repr__(self):
return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (

8
ogn/model/airport.py
Wyświetl plik

@ -11,16 +11,16 @@ class Airport(Base):
id = Column(Integer, primary_key=True)
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Integer)
altitude = Column(Float(precision=2))
name = Column(String, index=True)
code = Column(String(6))
country_code = Column(String(2))
style = Column(SmallInteger)
description = Column(String)
runway_direction = Column(Integer)
runway_length = Column(Integer)
frequency = Column(Float)
runway_direction = Column(SmallInteger)
runway_length = Column(SmallInteger)
frequency = Column(Float(precision=2))
def __repr__(self):
return "<Airport %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,% s>" % (

4
ogn/model/beacon.py
Wyświetl plik

@ -1,6 +1,6 @@
from geoalchemy2.shape import to_shape
from geoalchemy2.types import Geometry
from sqlalchemy import Column, String, Integer, Float, DateTime
from sqlalchemy import Column, String, Integer, SmallInteger, Float, DateTime
from sqlalchemy.ext.declarative import AbstractConcreteBase
from .base import Base
@ -20,7 +20,7 @@ class Beacon(AbstractConcreteBase, Base):
timestamp = Column(DateTime, index=True)
symboltable = None
symbolcode = None
track = Column(Integer)
track = Column(SmallInteger)
ground_speed = Column(Float(precision=2))
comment = None

2
ogn/model/device.py
Wyświetl plik

@ -14,7 +14,7 @@ class Device(Base):
lastseen = Column(DateTime, index=True)
aircraft_type = Column(SmallInteger, index=True)
stealth = Column(Boolean)
software_version = Column(Float)
software_version = Column(Float(precision=2))
hardware_version = Column(SmallInteger)
real_address = Column(String(6))

10
ogn/model/device_stats.py
Wyświetl plik

@ -10,17 +10,21 @@ class DeviceStats(Base):
id = Column(Integer, primary_key=True)
date = Column(Date)
max_altitude = Column(Float)
receiver_count = Column(Integer)
# Statistic data
max_altitude = Column(Float(precision=2))
receiver_count = Column(SmallInteger)
aircraft_beacon_count = Column(Integer)
firstseen = Column(DateTime)
lastseen = Column(DateTime)
aircraft_type = Column(SmallInteger)
stealth = Column(Boolean)
software_version = Column(Float)
software_version = Column(Float(precision=2))
hardware_version = Column(SmallInteger)
real_address = Column(String(6))
ambiguous = Column(Boolean)
# Relations
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id], backref='stats')

8
ogn/model/logbook.py
Wyświetl plik

@ -1,4 +1,4 @@
from sqlalchemy import Integer, DateTime, Column, ForeignKey, case, null
from sqlalchemy import Integer, SmallInteger, Float, DateTime, Column, ForeignKey, case, null
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
@ -12,10 +12,10 @@ class Logbook(Base):
reftime = Column(DateTime, index=True)
takeoff_timestamp = Column(DateTime)
takeoff_track = Column(Integer)
takeoff_track = Column(SmallInteger)
landing_timestamp = Column(DateTime)
landing_track = Column(Integer)
max_altitude = Column(Integer)
landing_track = Column(SmallInteger)
max_altitude = Column(Float(precision=2))
# Relations
takeoff_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True)

4
ogn/model/receiver.py
Wyświetl plik

@ -1,6 +1,6 @@
from geoalchemy2.shape import to_shape
from geoalchemy2.types import Geometry
from sqlalchemy import Column, String, Integer, DateTime
from sqlalchemy import Column, Float, String, Integer, DateTime
from sqlalchemy.orm import relationship
from .base import Base
@ -13,7 +13,7 @@ class Receiver(Base):
id = Column(Integer, primary_key=True)
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Integer)
altitude = Column(Float(precision=2))
name = Column(String(9))
firstseen = Column(DateTime, index=True)

8
ogn/model/receiver_coverage.py
Wyświetl plik

@ -1,4 +1,4 @@
from sqlalchemy import Column, String, Integer, Float, Date, ForeignKey, Index
from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index
from sqlalchemy.orm import relationship
@ -13,11 +13,11 @@ class ReceiverCoverage(Base):
date = Column(Date, primary_key=True)
max_signal_quality = Column(Float)
max_altitude = Column(Integer)
min_altitude = Column(Integer)
max_altitude = Column(Float(precision=2))
min_altitude = Column(Float(precision=2))
aircraft_beacon_count = Column(Integer)
device_count = Column(Integer)
device_count = Column(SmallInteger)
# Relations
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_coverages')

14
ogn/model/receiver_stats.py
Wyświetl plik

@ -1,5 +1,6 @@
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String
from sqlalchemy.orm import relationship
from geoalchemy2.types import Geometry
from .base import Base
@ -10,10 +11,17 @@ class ReceiverStats(Base):
id = Column(Integer, primary_key=True)
date = Column(Date)
# Statistic data
aircraft_beacon_count = Column(Integer)
receiver_beacon_count = Column(Integer)
aircraft_count = Column(Integer)
aircraft_count = Column(SmallInteger)
max_distance = Column(Float)
firstseen = Column(DateTime, index=True)
lastseen = Column(DateTime, index=True)
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Float(precision=2))
version = Column(String)
platform = Column(String)
# Relations
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)

4
ogn/model/takeoff_landing.py
Wyświetl plik

@ -1,4 +1,4 @@
from sqlalchemy import Boolean, Column, Integer, DateTime, ForeignKey
from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base
@ -12,7 +12,7 @@ class TakeoffLanding(Base):
timestamp = Column(DateTime, primary_key=True)
is_takeoff = Column(Boolean)
track = Column(Integer)
track = Column(SmallInteger)
# Relations
airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings')