mirror
/
ogn-python
kopia lustrzana https://github.com/glidernet/ogn-python
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Speed up SQL query and ignore relayed beacons for receiver stats

pull/68/head
Konstantin Gründger 2019-02-16 10:21:17 +01:00
rodzic b30c7a423e
commit 7783840dc1
1 zmienionych plików z 15 dodań i 13 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

28
ogn_python/collect/stats.py
Wyświetl plik

@ -1,6 +1,6 @@
from celery.utils.log import get_task_logger from celery.utils.log import get_task_logger
from sqlalchemy import insert, distinct, between from sqlalchemy import insert, distinct, between, literal
from sqlalchemy.sql import null, and_, func, or_, update from sqlalchemy.sql import null, and_, func, or_, update
from sqlalchemy.sql.expression import case from sqlalchemy.sql.expression import case
@ -12,6 +12,9 @@ from ogn_python.utils import date_to_timestamps
logger = get_task_logger(__name__) logger = get_task_logger(__name__)
# 40dB@10km is enough for 640km
MAX_PLAUSIBLE_QUALITY = 40
@app.task @app.task
def create_device_stats(session=None, date=None): def create_device_stats(session=None, date=None):
@ -43,7 +46,7 @@ def create_device_stats(session=None, date=None):
# Calculate stats, firstseen, lastseen and last values != NULL # Calculate stats, firstseen, lastseen and last values != NULL
device_stats = session.query( device_stats = session.query(
distinct(sq.c.device_id).label('device_id'), distinct(sq.c.device_id).label('device_id'),
func.date(sq.c.timestamp).label('date'), literal(date).label('date'),
func.max(sq.c.dr) func.max(sq.c.dr)
.over(partition_by=sq.c.device_id) .over(partition_by=sq.c.device_id)
.label('receiver_count'), .label('receiver_count'),
@ -115,7 +118,7 @@ def create_receiver_stats(session=None, date=None):
# Calculate stats, firstseen, lastseen and last values != NULL # Calculate stats, firstseen, lastseen and last values != NULL
receiver_stats = session.query( receiver_stats = session.query(
distinct(sq.c.receiver_id).label('receiver_id'), distinct(sq.c.receiver_id).label('receiver_id'),
func.date(sq.c.timestamp).label('date'), literal(date).label('date'),
func.first_value(sq.c.timestamp) func.first_value(sq.c.timestamp)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()) .over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast())
.label('firstseen'), .label('firstseen'),
@ -145,21 +148,20 @@ def create_receiver_stats(session=None, date=None):
session.commit() session.commit()
logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
# Update aircraft_beacon_count, aircraft_count and max_distance (without any error and max quality of 36dB@10km which is enough for 640km ... ) # Update aircraft_beacon_count, aircraft_count and max_distance
aircraft_beacon_stats = session.query(func.date(AircraftBeacon.timestamp).label('date'), aircraft_beacon_stats = session.query(AircraftBeacon.receiver_id,
AircraftBeacon.receiver_id,
func.count(AircraftBeacon.timestamp).label('aircraft_beacon_count'), func.count(AircraftBeacon.timestamp).label('aircraft_beacon_count'),
func.count(func.distinct(AircraftBeacon.device_id)).label('aircraft_count'), func.count(func.distinct(AircraftBeacon.device_id)).label('aircraft_count'),
func.max(AircraftBeacon.distance).label('max_distance')) \ func.max(AircraftBeacon.distance).label('max_distance')) \
.filter(and_(between(AircraftBeacon.timestamp, start, end), .filter(and_(between(AircraftBeacon.timestamp, start, end),
AircraftBeacon.error_count == 0, AircraftBeacon.error_count == 0,
AircraftBeacon.quality <= 40)) \ AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY,
.group_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.relay == null())) \
AircraftBeacon.receiver_id) \ .group_by(AircraftBeacon.receiver_id) \
.subquery() .subquery()
upd = update(ReceiverStats) \ upd = update(ReceiverStats) \
.where(and_(ReceiverStats.date == aircraft_beacon_stats.c.date, .where(and_(ReceiverStats.date == date,
ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id)) \ ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id)) \
.values({'aircraft_beacon_count': aircraft_beacon_stats.c.aircraft_beacon_count, .values({'aircraft_beacon_count': aircraft_beacon_stats.c.aircraft_beacon_count,
'aircraft_count': aircraft_beacon_stats.c.aircraft_count, 'aircraft_count': aircraft_beacon_stats.c.aircraft_count,
@ -252,7 +254,7 @@ def create_relation_stats(session=None, date=None):
# Calculate stats for selected day # Calculate stats for selected day
relation_stats = session.query( relation_stats = session.query(
func.date(AircraftBeacon.timestamp), literal(date),
AircraftBeacon.device_id, AircraftBeacon.device_id,
AircraftBeacon.receiver_id, AircraftBeacon.receiver_id,
func.max(AircraftBeacon.quality), func.max(AircraftBeacon.quality),
@ -261,9 +263,9 @@ def create_relation_stats(session=None, date=None):
.filter(and_(between(AircraftBeacon.timestamp, start, end), .filter(and_(between(AircraftBeacon.timestamp, start, end),
AircraftBeacon.distance > 1000, AircraftBeacon.distance > 1000,
AircraftBeacon.error_count == 0, AircraftBeacon.error_count == 0,
AircraftBeacon.quality <= 40, AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY,
AircraftBeacon.ground_speed > 10)) \ AircraftBeacon.ground_speed > 10)) \
.group_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.receiver_id) \ .group_by(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id) \
.subquery() .subquery()
# And insert them # And insert them