Better doc and small improvements

README.md

@@ -124,10 +124,10 @@ available commands:
     show                   Show a logbook for <airport_name>.
   
   [stats]
-    add_missing_devices    Update devices with data from stats.
-    add_missing_receivers  Update receivers with data from stats.
     create_flights         Create Flights.
     create_stats           Create DeviceStats, ReceiverStats and RelationStats.
+    update_devices         Update devices with data from stats.
+    update_receivers       Update receivers with data from stats.
 ```
 
 Only the command `logbook.compute` requires a running task server (celery) at the moment.

ogn/collect/stats.py

@@ -186,39 +186,43 @@ def update_device_stats_jumps(session=None, date=None):
     else:
         (start, end) = date_to_timestamps(date)
 
+    # speed limits in m/s (values above indicates a unplausible position / jump)
+    max_horizontal_speed = 1000
+    max_vertical_speed = 100
+    max_jumps = 10  # threshold for an 'ambiguous' device
+
+    # find consecutive positions for a device
     sq = session.query(AircraftBeacon.device_id,
-                       AircraftBeacon.timestamp.label('t0'),
-                       func.lead(AircraftBeacon.timestamp).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('t1'),
-                       AircraftBeacon.location_wkt.label('l0'),
-                       func.lead(AircraftBeacon.location_wkt).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('l1'),
-                       AircraftBeacon.altitude.label('a0'),
-                       func.lead(AircraftBeacon.altitude).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('a1')) \
+                       AircraftBeacon.timestamp,
+                       func.lead(AircraftBeacon.timestamp).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('timestamp_next'),
+                       AircraftBeacon.location_wkt,
+                       func.lead(AircraftBeacon.location_wkt).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('location_next'),
+                       AircraftBeacon.altitude,
+                       func.lead(AircraftBeacon.altitude).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('altitude_next')) \
         .filter(and_(between(AircraftBeacon.timestamp, start, end),
                      AircraftBeacon.error_count == 0)) \
         .subquery()
 
+    # calc vertial and horizontal speed between points
     sq2 = session.query(sq.c.device_id,
-                        (func.st_distancesphere(sq.c.l1, sq.c.l0) / (func.extract('epoch', sq.c.t1) - func.extract('epoch', sq.c.t0))).label('horizontal_speed'),
-                        ((sq.c.a1 - sq.c.a0) / (func.extract('epoch', sq.c.t1) - func.extract('epoch', sq.c.t0))).label('vertical_speed')) \
-        .filter(and_(sq.c.t0 != null(),
-                     sq.c.t1 != null(),
-                     sq.c.t0 < sq.c.t1)) \
+                        (func.st_distancesphere(sq.c.location_next, sq.c.location) / (func.extract('epoch', sq.c.timestamp_next) - func.extract('epoch', sq.c.timestamp))).label('horizontal_speed'),
+                        ((sq.c.altitude_next - sq.c.altitude) / (func.extract('epoch', sq.c.timestamp_next) - func.extract('epoch', sq.c.timestamp))).label('vertical_speed')) \
+        .filter(and_(sq.c.timestamp != null(),
+                     sq.c.timestamp_next != null(),
+                     sq.c.timestamp < sq.c.timestamp_next)) \
         .subquery()
 
+    # ... and find and count 'jumps'
     sq3 = session.query(sq2.c.device_id,
-                        case([(or_(func.abs(sq2.c.horizontal_speed) > 1000, func.abs(sq2.c.vertical_speed) > 100), 1)], else_=0).label('jump')) \
-        .subquery()
-
-    sq4 = session.query(sq3.c.device_id,
-                        func.sum(sq3.c.jump).label('jumps')) \
-        .group_by(sq3.c.device_id) \
+                        func.sum(case([(or_(func.abs(sq2.c.horizontal_speed) > max_horizontal_speed, func.abs(sq2.c.vertical_speed) > max_vertical_speed), 1)], else_=0)).label('jumps')) \
+        .group_by(sq2.c.device_id) \
         .subquery()
 
     upd = update(DeviceStats) \
         .where(and_(DeviceStats.date == date,
-                    DeviceStats.device_id == sq4.c.device_id)) \
-        .values({'ambiguous': sq4.c.jumps > 10,
-                 'jumps': sq4.c.jumps})
+                    DeviceStats.device_id == sq3.c.device_id)) \
+        .values({'ambiguous': sq3.c.jumps > max_jumps,
+                 'jumps': sq3.c.jumps})
 
     result = session.execute(upd)
     update_counter = result.rowcount

ogn/commands/database.py

@@ -20,8 +20,8 @@ def get_database_days(start, end):
         start = days_from_db[0].date()
         end = days_from_db[1].date()
     else:
-        start = datetime.strptime(start, "%Y-%m-%d")
-        end = datetime.strptime(end, "%Y-%m-%d")
+        start = datetime.strptime(start, "%Y-%m-%d").date()
+        end = datetime.strptime(end, "%Y-%m-%d").date()
 
     days = get_days(start, end)
 

ogn/commands/stats.py

@@ -1,9 +1,11 @@
+from datetime import datetime
+from tqdm import tqdm
 from manager import Manager
 from ogn.commands.dbutils import session
 from ogn.commands.database import get_database_days
 
 from ogn.collect.stats import create_device_stats, create_receiver_stats, create_relation_stats,\
-    update_qualities, update_receivers, update_devices,\
+    update_qualities, update_receivers as update_receivers_command, update_devices as update_devices_command,\
     update_device_stats_jumps
 
 manager = Manager()
@@ -15,36 +17,29 @@ def create_stats(start=None, end=None):
 
     days = get_database_days(start, end)
 
-    for single_date in days:
+    pbar = tqdm(days)
+    for single_date in pbar:
+        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
         result = create_device_stats(session=session, date=single_date)
-        print(result)
-
         result = update_device_stats_jumps(session=session, date=single_date)
-        print(result)
-
         result = create_receiver_stats(session=session, date=single_date)
-        print(result)
-
         result = create_relation_stats(session=session, date=single_date)
-        print(result)
-
         result = update_qualities(session=session, date=single_date)
-        print(result)
 
 
 @manager.command
-def add_missing_receivers():
+def update_receivers():
     """Update receivers with data from stats."""
 
-    result = update_receivers(session=session)
+    result = update_receivers_command(session=session)
     print(result)
 
 
 @manager.command
-def add_missing_devices():
+def update_devices():
     """Update devices with data from stats."""
 
-    result = update_devices(session=session)
+    result = update_devices_command(session=session)
     print(result)
 
 
@@ -54,9 +49,10 @@ def create_flights(start=None, end=None):
 
     days = get_database_days(start, end)
 
-    for single_date in days:
+    pbar = tqdm(days)
+    for single_date in pbar:
+        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
         result = _create_flights2d(session=session, date=single_date)
-        print(result)
 
 
 def _create_flights2d(session=None, date=None):