From f9945770e9eef9dbfdded1bbcb6c1865bf6894ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 10 Jan 2019 21:33:52 +0100 Subject: [PATCH] Faster ogn range calculation --- ogn/collect/ognrange.py | 7 +++---- ogn/model/receiver_coverage.py | 7 ++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ogn/collect/ognrange.py b/ogn/collect/ognrange.py index 1b87b05..d7b6581 100644 --- a/ogn/collect/ognrange.py +++ b/ogn/collect/ognrange.py @@ -3,6 +3,7 @@ from celery.utils.log import get_task_logger from sqlalchemy import String from sqlalchemy import and_, insert, update, exists, between from sqlalchemy.sql import func, null +from sqlalchemy.sql.expression import literal_column from ogn.collect.celery import app from ogn.model import AircraftBeacon, ReceiverCoverage @@ -29,7 +30,6 @@ def create_receiver_coverage(session=None, date=None): # Filter aircraft beacons and shrink precision of MGRS from 1m to 1km resolution: 30UXC 00061 18429 -> 30UXC 00 18 sq = session.query((func.left(AircraftBeacon.location_mgrs, 5, type_=String) + func.substring(AircraftBeacon.location_mgrs, 6, 2, type_=String) + func.substring(AircraftBeacon.location_mgrs, 11, 2, type_=String)).label('reduced_mgrs'), AircraftBeacon.receiver_id, - func.date(AircraftBeacon.timestamp).label('date'), AircraftBeacon.signal_quality, AircraftBeacon.altitude, AircraftBeacon.device_id) \ @@ -42,15 +42,14 @@ def create_receiver_coverage(session=None, date=None): # ... and group them by reduced MGRS, receiver and date query = session.query(sq.c.reduced_mgrs, sq.c.receiver_id, - sq.c.date, + literal_column("'{}'".format(date.strftime('%Y-%m-%d'))).label('date'), func.max(sq.c.signal_quality).label('max_signal_quality'), func.min(sq.c.altitude).label('min_altitude'), func.max(sq.c.altitude).label('max_altitude'), func.count(sq.c.altitude).label('aircraft_beacon_count'), func.count(func.distinct(sq.c.device_id)).label('device_count')) \ .group_by(sq.c.reduced_mgrs, - sq.c.receiver_id, - sq.c.date) \ + sq.c.receiver_id) \ .subquery() # if a receiver coverage entry exist --> update it diff --git a/ogn/model/receiver_coverage.py b/ogn/model/receiver_coverage.py index 20f345a..873ab8c 100644 --- a/ogn/model/receiver_coverage.py +++ b/ogn/model/receiver_coverage.py @@ -1,5 +1,6 @@ -from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey +from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index from sqlalchemy.orm import relationship, backref +from sqlalchemy.sql import func from .base import Base @@ -21,3 +22,7 @@ class ReceiverCoverage(Base): # Relations receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), primary_key=True) receiver = relationship('Receiver', foreign_keys=[receiver_id], backref=backref('receiver_coverages', order_by='ReceiverCoverage.date.asc()')) + + +Index('ix_receiver_coverages_date_receiver_id', ReceiverCoverage.date, ReceiverCoverage.receiver_id) +Index('ix_receiver_coverages_receiver_id_date', ReceiverCoverage.receiver_id, ReceiverCoverage.date)