Merge pull request #1 from Meisterschueler/logbook

Logbook
pull/1/head
Meisterschueler 2015-11-11 08:14:44 +01:00
commit 95c8f5037e
10 zmienionych plików z 271 dodań i 98 usunięć

63
ogn/db_analysis.py 100644
Wyświetl plik

@ -0,0 +1,63 @@
from datetime import datetime, timedelta
from sqlalchemy.sql import func
from sqlalchemy import distinct, and_
from ogn.db import session
from ogn.model import Receiver
back_24h = datetime.utcnow() - timedelta(days=1)
receiver_messages_per_24h = 24*60 / 5
def get_receiver_info():
sq = session.query(distinct(Receiver.name).label('name'), func.max(Receiver.timestamp).label('lastseen'), func.count(Receiver.name).label('messages_count')).\
filter(Receiver.timestamp > back_24h).\
group_by(Receiver.name).\
subquery()
query = session.query(Receiver, sq.c.messages_count).\
filter(and_(Receiver.name == sq.c.name, Receiver.timestamp == sq.c.lastseen)).\
order_by(Receiver.name)
print('--- Receivers ---')
for [receiver, messages_count] in query.all():
print('%9s: %3d%% avail, %s, %s ' % (receiver.name, 100.0*float(messages_count/receiver_messages_per_24h), receiver.version, receiver.platform))
def get_software_stats():
sq = session.query(Receiver.name, func.max(Receiver.timestamp).label('lastseen')).\
filter(Receiver.timestamp > back_24h).\
group_by(Receiver.name).\
subquery()
versions = session.query(distinct(Receiver.version), func.count(Receiver.version)).\
filter(and_(Receiver.name == sq.c.name, Receiver.timestamp == sq.c.lastseen)).\
group_by(Receiver.version).\
order_by(Receiver.version)
print('\n--- Versions ---')
for [version, count] in versions.all():
print('%5s: %s' % (version, count))
def get_hardware_stats():
sq = session.query(Receiver.name, func.max(Receiver.timestamp).label('lastseen')).\
filter(Receiver.timestamp > back_24h).\
group_by(Receiver.name).\
subquery()
platforms = session.query(distinct(Receiver.platform), func.count(Receiver.platform)).\
filter(and_(Receiver.name == sq.c.name, Receiver.timestamp == sq.c.lastseen)).\
group_by(Receiver.platform).\
order_by(Receiver.platform)
print('\n--- Platforms ---')
for [platform, count] in platforms.all():
print('%7s: %s' % (platform, count))
if __name__ == '__main__':
get_receiver_info()
get_software_stats()
get_hardware_stats()

Wyświetl plik

@ -1,6 +1,6 @@
from ogn.db import session
from ogn.model import Flarm
from ogn.ognutils import get_devices_from_ddb
from ogn.ognutils import get_ddb
def put_into_db(beacon):
@ -10,8 +10,13 @@ def put_into_db(beacon):
def fill_flarm_db():
session.query(Flarm).delete()
flarms = get_devices_from_ddb()
flarms = get_ddb()
session.bulk_save_objects(flarms)
flarms = get_ddb('custom.txt')
session.bulk_save_objects(flarms)
session.commit()
if __name__ == '__main__':

139
ogn/logbook.py 100644
Wyświetl plik

@ -0,0 +1,139 @@
from datetime import datetime, timedelta
from sqlalchemy.sql import func, null
from sqlalchemy import and_, or_, insert, between
from sqlalchemy.sql.expression import case, true, false, label
from ogn.db import session
from ogn.model import Flarm, Position, TakeoffLanding
def compute_takeoff_and_landing():
takeoff_speed = 30
landing_speed = 30
# get last takeoff_landing time as starting point for the following search
last_takeoff_landing_query = session.query(func.max(TakeoffLanding.timestamp))
last_takeoff_landing = last_takeoff_landing_query.one()[0]
if last_takeoff_landing is None:
last_takeoff_landing = datetime(2015, 1, 1, 0, 0, 0)
# make a query with current, previous and next position, so we can detect takeoffs and landings
sq = session.query(Position.address,
func.lag(Position.address).over(order_by=and_(Position.address, Position.timestamp)).label('address_prev'),
func.lead(Position.address).over(order_by=and_(Position.address, Position.timestamp)).label('address_next'),
Position.timestamp,
func.lag(Position.timestamp).over(order_by=and_(Position.address, Position.timestamp)).label('timestamp_prev'),
func.lead(Position.timestamp).over(order_by=and_(Position.address, Position.timestamp)).label('timestamp_next'),
Position.latitude,
func.lag(Position.latitude).over(order_by=and_(Position.address, Position.timestamp)).label('latitude_prev'),
func.lead(Position.latitude).over(order_by=and_(Position.address, Position.timestamp)).label('latitude_next'),
Position.longitude,
func.lag(Position.longitude).over(order_by=and_(Position.address, Position.timestamp)).label('longitude_prev'),
func.lead(Position.longitude).over(order_by=and_(Position.address, Position.timestamp)).label('longitude_next'),
Position.ground_speed,
Position.track,
func.lag(Position.track).over(order_by=and_(Position.address, Position.timestamp)).label('track_prev'),
func.lead(Position.track).over(order_by=and_(Position.address, Position.timestamp)).label('track_next'),
Position.ground_speed,
func.lag(Position.ground_speed).over(order_by=and_(Position.address, Position.timestamp)).label('ground_speed_prev'),
func.lead(Position.ground_speed).over(order_by=and_(Position.address, Position.timestamp)).label('ground_speed_next'),
Position.altitude,
func.lag(Position.altitude).over(order_by=and_(Position.address, Position.timestamp)).label('altitude_prev'),
func.lead(Position.altitude).over(order_by=and_(Position.address, Position.timestamp)).label('altitude_next')) \
.filter(Position.timestamp > last_takeoff_landing) \
.order_by(func.date(Position.timestamp), Position.address, Position.timestamp) \
.subquery()
# find takeoffs and landings (look at the trigger_speed)
takeoff_landing_query = session.query(sq.c.address, sq.c.timestamp, sq.c.latitude, sq.c.longitude, sq.c.track, sq.c.ground_speed, sq.c.altitude, case([(sq.c.ground_speed>takeoff_speed, True), (sq.c.ground_speed<landing_speed, False)]).label('is_takeoff')) \
.filter(sq.c.address_prev == sq.c.address == sq.c.address_next) \
.filter(or_(and_(sq.c.ground_speed_prev < takeoff_speed, # takeoff
sq.c.ground_speed > takeoff_speed,
sq.c.ground_speed_next > takeoff_speed),
and_(sq.c.ground_speed_prev > landing_speed, # landing
sq.c.ground_speed < landing_speed,
sq.c.ground_speed_next < landing_speed))) \
.order_by(func.date(sq.c.timestamp), sq.c.timestamp)
# ... and save them
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
session.execute(ins)
session.commit()
def get_logbook(airport_name, latitude, longitude, altitude):
latmin = latitude - 0.15
latmax = latitude + 0.15
lonmin = longitude - 0.15
lonmax = longitude + 0.15
max_altitude = altitude + 200
# make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
sq = session.query(TakeoffLanding.address,
func.lag(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_prev'),
func.lead(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_next'),
TakeoffLanding.timestamp,
func.lag(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_prev'),
func.lead(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_next'),
TakeoffLanding.track,
func.lag(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_prev'),
func.lead(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_next'),
TakeoffLanding.is_takeoff,
func.lag(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_prev'),
func.lead(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_next')) \
.filter(and_(between(TakeoffLanding.latitude, latmin, latmax), between(TakeoffLanding.longitude, lonmin, lonmax))) \
.filter(TakeoffLanding.altitude < max_altitude) \
.subquery()
# find complete flights (with takeoff and landing) with duration < 1 day
complete_flight_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), sq.c.timestamp_next.label('landing'), sq.c.track_next.label('landing_track'), label('duration', sq.c.timestamp_next - sq.c.timestamp)) \
.filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) \
.filter(sq.c.address == sq.c.address_next) \
.filter(sq.c.timestamp_next - sq.c.timestamp < timedelta(days=1))
# split complete flights (with takeoff and landing) with duration > 1 day into one takeoff and one landing
split_start_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'), null().label('duration')) \
.filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) \
.filter(sq.c.address == sq.c.address_next) \
.filter(sq.c.timestamp_next - sq.c.timestamp >= timedelta(days=1))
split_landing_query = session.query(sq.c.timestamp_next.label('reftime'), sq.c.address.label('address'), null().label('takeoff'), null().label('takeoff_track'), sq.c.timestamp_next.label('landing'), sq.c.track_next.label('landing_track'), null().label('duration')) \
.filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) \
.filter(sq.c.address == sq.c.address_next) \
.filter(sq.c.timestamp_next - sq.c.timestamp >= timedelta(days=1))
# find landings without start
only_landings_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), null().label('takeoff'), null().label('takeoff_track'), sq.c.timestamp.label('landing'), sq.c.track_next.label('landing_track'), null().label('duration')) \
.filter(sq.c.is_takeoff == false()) \
.filter(or_(sq.c.address != sq.c.address_prev,
sq.c.is_takeoff_prev == false()))
# find starts without landing
only_starts_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'), null().label('duration')) \
.filter(sq.c.is_takeoff == true()) \
.filter(or_(sq.c.address != sq.c.address_next,
sq.c.is_takeoff_next == true()))
# unite all
union_query = complete_flight_query.union(split_start_query, split_landing_query, only_landings_query, only_starts_query) \
.subquery()
# get aircraft informations and sort all entries by the reference time
logbook_query = session.query(union_query.c.reftime, union_query.c.address, union_query.c.takeoff, union_query.c.takeoff_track, union_query.c.landing, union_query.c.landing_track, union_query.c.duration, Flarm.registration, Flarm.aircraft) \
.outerjoin(Flarm, union_query.c.address == Flarm.address) \
.order_by(union_query.c.reftime)
print('--- Logbook (' + airport_name + ') ---')
none_datetime_replacer = lambda datetime_object: '--:--:--' if datetime_object is None else datetime_object.time()
none_track_replacer = lambda track_object: '--' if track_object is None else round(track_object/10.0)
none_timedelta_replacer = lambda timedelta_object: '--:--:--' if timedelta_object is None else timedelta_object
none_registration_replacer = lambda registration_object, address: '[' + address + ']' if registration_object is None else registration_object
none_aircraft_replacer = lambda aircraft_object: '(unknown)' if aircraft_object is None else aircraft_object
for [reftime, address, takeoff, takeoff_track, landing, landing_track, duration, registration, aircraft] in logbook_query.all():
print('%10s %8s (%2s) %8s (%2s) %8s %8s %s' % (reftime.date(), none_datetime_replacer(takeoff), none_track_replacer(takeoff_track), none_datetime_replacer(landing), none_track_replacer(landing_track), none_timedelta_replacer(duration), none_registration_replacer(registration, address), none_aircraft_replacer(aircraft)))
if __name__ == '__main__':
compute_takeoff_and_landing()
get_logbook('Königsdorf', 47.83, 11.46, 601)

Wyświetl plik

@ -5,3 +5,4 @@ from .beacon import Beacon
from .flarm import Flarm
from .position import Position
from .receiver import Receiver
from .takeoff_landing import TakeoffLanding

Wyświetl plik

@ -1,8 +1,5 @@
import re
from sqlalchemy import Column, Integer, String, Unicode, Boolean, SmallInteger
from .address_origin import AddressOrigin
from .base import Base
@ -23,46 +20,5 @@ class Flarm(Base):
address_origin = Column(SmallInteger)
FLARMNET_LINE_LENGTH = 173
def parse_ogn(self, line):
PATTERN = "\'([FIO])\',\'(.{6})\',\'([^\']+)?\',\'([^\']+)?\',\'([^\']+)?\',\'([YN])\',\'([YN])\'"
ogn_re = re.compile(PATTERN)
result = ogn_re.match(line)
if result is None:
raise Exception("No valid string: %s" % line)
self.address_type = result.group(1)
self.address = result.group(2)
self.aircraft = result.group(3)
self.registration = result.group(4)
self.competition = result.group(5)
self.tracked = result.group(6) == "Y"
self.identified = result.group(7) == "Y"
self.address_origin = AddressOrigin.ogn_ddb
def parse_flarmnet(self, line):
rawString = self.hexToString(line)
self.address_type = None
self.address = rawString[0:6].strip()
self.name = rawString[6:27].strip()
self.airport = rawString[27:48].strip()
self.aircraft = rawString[48:69].strip()
self.registration = rawString[69:76].strip()
self.competition = rawString[76:79].strip()
self.frequency = rawString[79:89].strip()
self.address_origin = AddressOrigin.flarmnet
def hexToString(self, hexString):
result = ''
for i in range(0, len(hexString)-1, 2):
result += chr(int(hexString[i:i+2], 16))
return(result)
def __repr__(self):
return("<Flarm: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.address_type, self.address, self.name, self.airport, self.aircraft, self.registration, self.competition, self.frequency, self.tracked, self.identified))

Wyświetl plik

@ -0,0 +1,10 @@
from sqlalchemy import Column, String, Boolean
from .beacon import Beacon
class TakeoffLanding(Beacon):
__tablename__ = 'takeoff_landing'
address = Column(String(6), index=True)
is_takeoff = Column(Boolean)

Wyświetl plik

@ -1,21 +1,39 @@
from urllib.request import urlopen
import requests
import csv
from io import StringIO
from ogn.model import Flarm
from .model import Flarm, AddressOrigin
from geopy.geocoders import Nominatim
DDB_URL = "http://ddb.glidernet.org/download"
def get_ddb(csvfile=None):
if csvfile is None:
r = requests.get(DDB_URL)
rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
address_origin = AddressOrigin.ogn_ddb
else:
r = open(csvfile, 'r')
rows = ''.join(i for i in r.readlines() if i[0] != '#')
address_origin = AddressOrigin.userdefined
data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
def get_devices_from_ddb():
devices = list()
response = urlopen("http://ddb.glidernet.org/download")
lines = response.readlines()
for line in lines:
if (line.decode()[0] == "#"):
continue
for row in data:
flarm = Flarm()
flarm.parse_ogn(line.decode())
flarm.address_type = row[0]
flarm.address = row[1]
flarm.aircraft = row[2]
flarm.registration = row[3]
flarm.competition = row[4]
flarm.tracked = row[5] == 'Y'
flarm.identified = row[6] == 'Y'
flarm.address_origin = address_origin
devices.append(flarm)
return devices

Wyświetl plik

@ -0,0 +1,4 @@
#DEVICE_TYPE,DEVICE_ID,AIRCRAFT_MODEL,REGISTRATION,CN,TRACKED,IDENTIFIED
'F','DD4711','HK36 TTC','D-EULE','','Y','Y'
'F','DD0815','Ventus 2cxM','D-1234','','Y','Y'
'F','DD3141','Arcus T','OE-4321','','Y','Y'

Wyświetl plik

@ -1,38 +0,0 @@
import unittest
from ogn.model.address_origin import *
from ogn.model.flarm import Flarm
class TestStringMethods(unittest.TestCase):
def test_ddb(self):
flarm = Flarm()
flarm.parse_ogn("'F','DD9703','Twin Astir II','D-8203','7G','Y','N'\r\n")
self.assertEqual(flarm.address_type, 'F')
self.assertEqual(flarm.address, 'DD9703')
self.assertEqual(flarm.aircraft, 'Twin Astir II')
self.assertEqual(flarm.registration, 'D-8203')
self.assertEqual(flarm.competition, '7G')
self.assertTrue(flarm.tracked)
self.assertFalse(flarm.identified)
self.assertEqual(flarm.address_origin, AddressOrigin.ogn_ddb)
def test_flarmnet(self):
flarm = Flarm()
flarm.parse_flarmnet('444431323334486972616d205965616765722020202020202020204c535a46202020202020202020202020202020202056656e747573203263784d2020202020202020202052552d343731315836203132332e343536')
self.assertEqual(flarm.address, 'DD1234')
self.assertEqual(flarm.name, 'Hiram Yeager')
self.assertEqual(flarm.airport, 'LSZF')
self.assertEqual(flarm.aircraft, 'Ventus 2cxM')
self.assertEqual(flarm.registration, 'RU-4711')
self.assertEqual(flarm.competition, 'X6')
self.assertEqual(flarm.frequency, '123.456')
self.assertEqual(flarm.address_origin, AddressOrigin.flarmnet)
if __name__ == '__main__':
unittest.main()

Wyświetl plik

@ -1,12 +1,27 @@
import unittest
from ogn.ognutils import get_devices_from_ddb, get_country_code
from ogn.ognutils import get_ddb, get_country_code
from ogn.model.address_origin import AddressOrigin
class TestStringMethods(unittest.TestCase):
def test_get_devices_from_ddb(self):
devices = get_devices_from_ddb()
def test_get_devices(self):
devices = get_ddb()
self.assertGreater(len(devices), 1000)
def test_get_ddb_from_file(self):
devices = get_ddb('tests/custom_ddb.txt')
self.assertEqual(len(devices), 3)
device = devices[0]
self.assertEqual(device.address, 'DD4711')
self.assertEqual(device.aircraft, 'HK36 TTC')
self.assertEqual(device.registration, 'D-EULE')
self.assertEqual(device.competition, '')
self.assertTrue(device.tracked)
self.assertTrue(device.identified)
self.assertEqual(device.address_origin, AddressOrigin.userdefined)
def test_get_country_code(self):
latitude = 48.0
longitude = 11.0