Cleanup

README.md

@@ -78,8 +78,10 @@ The task server must be running for `db.updateddb`.
 - [x] Document/Improve cli commands
 - [ ] Separate settings from module (currently at ogn/command/dbutils.py)
 - [ ] Enable granular data acquisition (eg. store receiver beacons only)
-- [ ] Database: Rename 'Flarm' to 'Device'?
 - [ ] Future Database-Migrations: Use Alembric?
+  - [ ] Rename 'Flarm' to 'Device'?
+  - [ ] Rename self.heared\_aircraft\_IDs (lowercase) in aircraft\_beacon
+  - [ ] Rename self.heared\_aircraft\_IDs
 - [ ] Fix command/logbook.py (@Meisterschueler?)
 - [ ] Introduce scheduled tasks with 'celery beat' (eg. updateddb)
 

ogn/aprs_utils.py

@@ -6,16 +6,16 @@ kmh2kts = 0.539957
 feet2m = 0.3048
 ms2fpm = 196.85
 
-kts2kmh = 1/kmh2kts
+kts2kmh = 1 / kmh2kts
-m2feet = 1/feet2m
+m2feet = 1 / feet2m
-fpm2ms = 1/ms2fpm
+fpm2ms = 1 / ms2fpm
 
 
 def dmsToDeg(dms):
     absDms = abs(dms)
     d = math.floor(absDms)
     m = (absDms - d) * 100 / 60
-    return (d + m)
+    return d + m
 
 
 def createTimestamp(hhmmss, reference):
@@ -31,6 +31,6 @@ def createTimestamp(hhmmss, reference):
         reference = reference + timedelta(days=1)
     elif (reference.hour == 0) & (hh == 23):
         reference = reference - timedelta(days=1)
-    elif (abs(reference.hour - hh) > 1):
+    elif abs(reference.hour - hh) > 1:
         raise Exception("Time difference is too big. Reference time:%s - timestamp:%s" % (reference, hhmmss))
     return datetime(reference.year, reference.month, reference.day, hh, mm, ss)

ogn/collect/celery.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from celery import Celery, Task
+from celery import Celery
 from celery.signals import worker_init, worker_shutdown
 
 app = Celery('ogn.collect',
@@ -13,6 +13,7 @@ app = Celery('ogn.collect',
 
 DB_URI = 'sqlite:///beacons.db'
 
+
 @worker_init.connect
 def connect_db(signal, sender):
     # Load settings like DB_URI...
@@ -21,6 +22,7 @@ def connect_db(signal, sender):
     Session = sessionmaker(bind=engine)
     sender.app.session = Session()
 
+
 @worker_shutdown.connect
 def close_db(signal, sender):
     sender.app.session.close()

ogn/collect/logbook.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from celery.utils.log import get_task_logger
 from ogn.collect.celery import app
@@ -9,7 +9,7 @@ from sqlalchemy.sql import func, null
 from sqlalchemy import and_, or_, insert, between
 from sqlalchemy.sql.expression import case, true, false, label
 
-from ogn.model import Flarm, AircraftBeacon, TakeoffLanding
+from ogn.model import AircraftBeacon, TakeoffLanding
 
 logger = get_task_logger(__name__)
 
@@ -26,34 +26,44 @@ def compute_takeoff_and_landing():
         last_takeoff_landing = datetime(2015, 1, 1, 0, 0, 0)
 
     # make a query with current, previous and next position, so we can detect takeoffs and landings
-    sq = app.session.query(AircraftBeacon.address,
+    sq = app.session.query(
-                       func.lag(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_prev'),
+        AircraftBeacon.address,
-                       func.lead(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_next'),
+        func.lag(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_prev'),
-                       AircraftBeacon.timestamp,
+        func.lead(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_next'),
-                       func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
+        AircraftBeacon.timestamp,
-                       func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
+        func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
-                       AircraftBeacon.latitude,
+        func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
-                       func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
+        AircraftBeacon.latitude,
-                       func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
+        func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
-                       AircraftBeacon.longitude,
+        func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
-                       func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
+        AircraftBeacon.longitude,
-                       func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
+        func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
-                       AircraftBeacon.ground_speed,
+        func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
-                       AircraftBeacon.track,
+        AircraftBeacon.ground_speed,
-                       func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
+        AircraftBeacon.track,
-                       func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
+        func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
-                       AircraftBeacon.ground_speed,
+        func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
-                       func.lag(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_prev'),
+        AircraftBeacon.ground_speed,
-                       func.lead(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_next'),
+        func.lag(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_prev'),
-                       AircraftBeacon.altitude,
+        func.lead(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_next'),
-                       func.lag(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_prev'),
+        AircraftBeacon.altitude,
-                       func.lead(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_next')) \
+        func.lag(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_prev'),
+        func.lead(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_next')) \
         .filter(AircraftBeacon.timestamp > last_takeoff_landing) \
         .order_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.address, AircraftBeacon.timestamp) \
         .subquery()
 
     # find takeoffs and landings (look at the trigger_speed)
-    takeoff_landing_query = app.session.query(sq.c.address, sq.c.timestamp, sq.c.latitude, sq.c.longitude, sq.c.track, sq.c.ground_speed, sq.c.altitude, case([(sq.c.ground_speed>takeoff_speed, True), (sq.c.ground_speed<landing_speed, False)]).label('is_takeoff')) \
+    takeoff_landing_query = app.session.query(
+        sq.c.address,
+        sq.c.timestamp,
+        sq.c.latitude,
+        sq.c.longitude,
+        sq.c.track,
+        sq.c.ground_speed,
+        sq.c.altitude,
+        case([(sq.c.ground_speed > takeoff_speed, True),
+              (sq.c.ground_speed < landing_speed, False)]).label('is_takeoff')) \
         .filter(sq.c.address_prev == sq.c.address == sq.c.address_next) \
         .filter(or_(and_(sq.c.ground_speed_prev < takeoff_speed,    # takeoff
                          sq.c.ground_speed > takeoff_speed,
@@ -67,6 +77,3 @@ def compute_takeoff_and_landing():
     ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
     app.session.execute(ins)
     app.session.commit()
-
-
-

ogn/commands/database.py

@@ -5,6 +5,7 @@ manager = Manager()
 
 from ogn.collect.fetchddb import update_ddb_data
 
+
 @manager.command
 def init():
     """Initialize the database."""

ogn/commands/logbook.py

@@ -1,18 +1,19 @@
 # -*- coding: utf-8 -*-
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from sqlalchemy.sql import func, null
 from sqlalchemy import and_, or_, insert, between
 from sqlalchemy.sql.expression import case, true, false, label
 
-from ogn.model import Flarm, AircraftBeacon, TakeoffLanding
+from ogn.model import Flarm, TakeoffLanding
 
 from ogn.commands.dbutils import session
 
 from manager import Manager
 manager = Manager()
 
+
 @manager.command
 def show(airport_name, latitude, longitude, altitude):
     """Show a logbook for <airport_name> located at given position."""
@@ -27,19 +28,62 @@ def show(airport_name, latitude, longitude, altitude):
     max_altitude = altitude + 200
 
     # make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
-    sq = session.query(TakeoffLanding.address,
+    sq = session.query(
-                       func.lag(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_prev'),
+        TakeoffLanding.address,
-                       func.lead(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_next'),
+        func.lag(TakeoffLanding.address) \
-                       TakeoffLanding.timestamp,
+            .over(
-                       func.lag(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_prev'),
+                order_by=and_(func.date(TakeoffLanding.timestamp),
-                       func.lead(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_next'),
+                              TakeoffLanding.address,
-                       TakeoffLanding.track,
+                              TakeoffLanding.timestamp)) \
-                       func.lag(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_prev'),
+            .label('address_prev'),
-                       func.lead(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_next'),
+        func.lead(TakeoffLanding.address) \
-                       TakeoffLanding.is_takeoff,
+            .over(
-                       func.lag(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_prev'),
+                order_by=and_(func.date(TakeoffLanding.timestamp),
-                       func.lead(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_next')) \
+                              TakeoffLanding.address,
-        .filter(and_(between(TakeoffLanding.latitude, latmin, latmax), between(TakeoffLanding.longitude, lonmin, lonmax))) \
+                              TakeoffLanding.timestamp)) \
+           .label('address_next'),
+       TakeoffLanding.timestamp,
+       func.lag(
+           TakeoffLanding.timestamp) \
+               .over(
+                   order_by=and_(func.date(TakeoffLanding.timestamp),
+                                 TakeoffLanding.address,
+                                 TakeoffLanding.timestamp)) \
+               .label('timestamp_prev'),
+       func.lead(
+           TakeoffLanding.timestamp) \
+               .over(order_by=and_(func.date(TakeoffLanding.timestamp),
+                                   TakeoffLanding.address,
+                                   TakeoffLanding.timestamp)) \
+               .label('timestamp_next'),
+       TakeoffLanding.track,
+       func.lag(
+           TakeoffLanding.track) \
+               .over(order_by=and_(func.date(TakeoffLanding.timestamp),
+                                   TakeoffLanding.address,
+                                   TakeoffLanding.timestamp)) \
+               .label('track_prev'),
+       func.lead(
+           TakeoffLanding.track) \
+               .over(order_by=and_(func.date(TakeoffLanding.timestamp),
+                                   TakeoffLanding.address,
+                                   TakeoffLanding.timestamp)) \
+               .label('track_next'),
+       TakeoffLanding.is_takeoff,
+       func.lag(
+           TakeoffLanding.is_takeoff) \
+               .over(order_by=and_(func.date(TakeoffLanding.timestamp),
+                                   TakeoffLanding.address,
+                                   TakeoffLanding.timestamp)) \
+               .label('is_takeoff_prev'),
+       func.lead(
+           TakeoffLanding.is_takeoff) \
+                .over(order_by=and_(func.date(TakeoffLanding.timestamp),
+                                    TakeoffLanding.address,
+                                    TakeoffLanding.timestamp)) \
+                .label('is_takeoff_next')) \
+        .filter(and_(between(TakeoffLanding.latitude, latmin, latmax),
+                     between(TakeoffLanding.longitude, lonmin, lonmax))) \
         .filter(TakeoffLanding.altitude < max_altitude) \
         .subquery()
 
@@ -50,7 +94,7 @@ def show(airport_name, latitude, longitude, altitude):
         .filter(sq.c.timestamp_next - sq.c.timestamp < timedelta(days=1))
 
     # split complete flights (with takeoff and landing) with duration > 1 day into one takeoff and one landing
-    split_start_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'),  null().label('duration')) \
+    split_start_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'), null().label('duration')) \
         .filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) \
         .filter(sq.c.address == sq.c.address_next) \
         .filter(sq.c.timestamp_next - sq.c.timestamp >= timedelta(days=1))
@@ -73,19 +117,42 @@ def show(airport_name, latitude, longitude, altitude):
                     sq.c.is_takeoff_next == true()))
 
     # unite all
-    union_query = complete_flight_query.union(split_start_query, split_landing_query, only_landings_query, only_starts_query) \
+    union_query = complete_flight_query.union(
+        split_start_query,
+        split_landing_query,
+        only_landings_query,
+        only_starts_query) \
         .subquery()
 
     # get aircraft informations and sort all entries by the reference time
-    logbook_query = session.query(union_query.c.reftime, union_query.c.address, union_query.c.takeoff, union_query.c.takeoff_track, union_query.c.landing, union_query.c.landing_track, union_query.c.duration, Flarm.registration, Flarm.aircraft) \
+    logbook_query = session.query(
+        union_query.c.reftime,
+        union_query.c.address,
+        union_query.c.takeoff,
+        union_query.c.takeoff_track,
+        union_query.c.landing,
+        union_query.c.landing_track,
+        union_query.c.duration,
+        Flarm.registration,
+        Flarm.aircraft) \
         .outerjoin(Flarm, union_query.c.address == Flarm.address) \
         .order_by(union_query.c.reftime)
 
-    print('--- Logbook (' + airport_name + ') ---')
+    print('--- Logbook (%s) ---' % airport_name)
+
     none_datetime_replacer = lambda datetime_object: '--:--:--' if datetime_object is None else datetime_object.time()
-    none_track_replacer = lambda track_object: '--' if track_object is None else round(track_object/10.0)
+    none_track_replacer = lambda track_object: '--' if track_object is None else round(track_object / 10.0)
     none_timedelta_replacer = lambda timedelta_object: '--:--:--' if timedelta_object is None else timedelta_object
     none_registration_replacer = lambda registration_object, address: '[' + address + ']' if registration_object is None else registration_object
     none_aircraft_replacer = lambda aircraft_object: '(unknown)' if aircraft_object is None else aircraft_object
+
     for [reftime, address, takeoff, takeoff_track, landing, landing_track, duration, registration, aircraft] in logbook_query.all():
-        print('%10s   %8s (%2s)   %8s (%2s)   %8s   %8s   %s' % (reftime.date(), none_datetime_replacer(takeoff), none_track_replacer(takeoff_track), none_datetime_replacer(landing), none_track_replacer(landing_track), none_timedelta_replacer(duration), none_registration_replacer(registration, address), none_aircraft_replacer(aircraft)))
+        print('%10s   %8s (%2s)   %8s (%2s)   %8s   %8s   %s' % (
+            reftime.date(),
+            none_datetime_replacer(takeoff),
+            none_track_replacer(takeoff_track),
+            none_datetime_replacer(landing),
+            none_track_replacer(landing_track),
+            none_timedelta_replacer(duration),
+            none_registration_replacer(registration, address),
+            none_aircraft_replacer(aircraft)))

ogn/commands/showreceiver.py

@@ -9,7 +9,7 @@ from ogn.model.receiver_device import ReceiverDevice
 from ogn.commands.dbutils import session
 
 back_24h = datetime.utcnow() - timedelta(days=1)
-receiver_messages_per_24h = 24*60 / 5
+receiver_messages_per_24h = 24 * 60 / 5
 
 from manager import Manager
 manager = Manager()

ogn/model/aircraft_beacon.py

@@ -97,7 +97,7 @@ class AircraftBeacon(Beacon):
                 self.stealth = ((int(address_match.group(1), 16) & 0b10000000) >> 7 == 1)
                 self.address = address_match.group(2)
             elif climb_rate_match is not None:
-                self.climb_rate = int(climb_rate_match.group(1))*fpm2ms
+                self.climb_rate = int(climb_rate_match.group(1)) * fpm2ms
             elif turn_rate_match is not None:
                 self.turn_rate = float(turn_rate_match.group(1))
             elif signal_strength_match is not None:
@@ -130,4 +130,18 @@ class AircraftBeacon(Beacon):
                 raise Exception("No valid position description: %s" % part)
 
     def __repr__(self):
-        return("<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.name, self.address_type, self.aircraft_type, self.timestamp, self.address_type, self.aircraft_type, self.stealth, self.address, self.climb_rate, self.turn_rate, self.signal_strength, self.error_count, self.frequency_offset, self.gps_status))
+        return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+            self.name,
+            self.address_type,
+            self.aircraft_type,
+            self.timestamp,
+            self.address_type,
+            self.aircraft_type,
+            self.stealth,
+            self.address,
+            self.climb_rate,
+            self.turn_rate,
+            self.signal_strength,
+            self.error_count,
+            self.frequency_offset,
+            self.gps_status)

ogn/model/flarm.py

@@ -21,4 +21,14 @@ class Flarm(Base):
     address_origin = Column(SmallInteger)
 
     def __repr__(self):
-        return("<Flarm: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.address_type, self.address, self.name, self.airport, self.aircraft, self.registration, self.competition, self.frequency, self.tracked, self.identified))
+        return "<Flarm: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+            self.address_type,
+            self.address,
+            self.name,
+            self.airport,
+            self.aircraft,
+            self.registration,
+            self.competition,
+            self.frequency,
+            self.tracked,
+            self.identified)

ogn/model/receiver_beacon.py

@@ -85,4 +85,4 @@ class ReceiverBeacon(Beacon):
                 raise Exception("No valid receiver description: %s" % part)
 
     def __repr__(self):
-        return("<ReceiverBeacon %s: %s>" % (self.name, self.version))
+        return "<ReceiverBeacon %s: %s>" % (self.name, self.version)