mirror
/
ogn-python
kopia lustrzana https://github.com/glidernet/ogn-python
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Cleanup

pull/1/head
Fabian P. Schmidt 2015-11-15 19:31:58 +01:00
rodzic d863aaf76d
commit bda6310ff6
10 zmienionych plików z 164 dodań i 61 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
README.md
Wyświetl plik

@ -78,8 +78,10 @@ The task server must be running for `db.updateddb`.
- [x] Document/Improve cli commands
- [ ] Separate settings from module (currently at ogn/command/dbutils.py)
- [ ] Enable granular data acquisition (eg. store receiver beacons only)
- [ ] Database: Rename 'Flarm' to 'Device'?
- [ ] Future Database-Migrations: Use Alembric?
- [ ] Rename 'Flarm' to 'Device'?
- [ ] Rename self.heared\_aircraft\_IDs (lowercase) in aircraft\_beacon
- [ ] Rename self.heared\_aircraft\_IDs
- [ ] Fix command/logbook.py (@Meisterschueler?)
- [ ] Introduce scheduled tasks with 'celery beat' (eg. updateddb)

10
ogn/aprs_utils.py
Wyświetl plik

@ -6,16 +6,16 @@ kmh2kts = 0.539957
feet2m = 0.3048
ms2fpm = 196.85
kts2kmh = 1/kmh2kts
m2feet = 1/feet2m
fpm2ms = 1/ms2fpm
kts2kmh = 1 / kmh2kts
m2feet = 1 / feet2m
fpm2ms = 1 / ms2fpm
def dmsToDeg(dms):
absDms = abs(dms)
d = math.floor(absDms)
m = (absDms - d) * 100 / 60
return (d + m)
return d + m
def createTimestamp(hhmmss, reference):
@ -31,6 +31,6 @@ def createTimestamp(hhmmss, reference):
reference = reference + timedelta(days=1)
elif (reference.hour == 0) & (hh == 23):
reference = reference - timedelta(days=1)
elif (abs(reference.hour - hh) > 1):
elif abs(reference.hour - hh) > 1:
raise Exception("Time difference is too big. Reference time:%s - timestamp:%s" % (reference, hhmmss))
return datetime(reference.year, reference.month, reference.day, hh, mm, ss)

4
ogn/collect/celery.py
Wyświetl plik

@ -3,7 +3,7 @@ from __future__ import absolute_import
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from celery import Celery, Task
from celery import Celery
from celery.signals import worker_init, worker_shutdown
app = Celery('ogn.collect',
@ -13,6 +13,7 @@ app = Celery('ogn.collect',
DB_URI = 'sqlite:///beacons.db'
@worker_init.connect
def connect_db(signal, sender):
# Load settings like DB_URI...
@ -21,6 +22,7 @@ def connect_db(signal, sender):
Session = sessionmaker(bind=engine)
sender.app.session = Session()
@worker_shutdown.connect
def close_db(signal, sender):
sender.app.session.close()

63
ogn/collect/logbook.py
Wyświetl plik

@ -1,6 +1,6 @@
from __future__ import absolute_import
from datetime import datetime, timedelta
from datetime import datetime
from celery.utils.log import get_task_logger
from ogn.collect.celery import app
@ -9,7 +9,7 @@ from sqlalchemy.sql import func, null
from sqlalchemy import and_, or_, insert, between
from sqlalchemy.sql.expression import case, true, false, label
from ogn.model import Flarm, AircraftBeacon, TakeoffLanding
from ogn.model import AircraftBeacon, TakeoffLanding
logger = get_task_logger(__name__)
@ -26,34 +26,44 @@ def compute_takeoff_and_landing():
last_takeoff_landing = datetime(2015, 1, 1, 0, 0, 0)
# make a query with current, previous and next position, so we can detect takeoffs and landings
sq = app.session.query(AircraftBeacon.address,
func.lag(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_prev'),
func.lead(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_next'),
AircraftBeacon.timestamp,
func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
AircraftBeacon.latitude,
func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
AircraftBeacon.longitude,
func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
AircraftBeacon.ground_speed,
AircraftBeacon.track,
func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
AircraftBeacon.ground_speed,
func.lag(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_prev'),
func.lead(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_next'),
AircraftBeacon.altitude,
func.lag(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_prev'),
func.lead(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_next')) \
sq = app.session.query(
AircraftBeacon.address,
func.lag(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_prev'),
func.lead(AircraftBeacon.address).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('address_next'),
AircraftBeacon.timestamp,
func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
AircraftBeacon.latitude,
func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
AircraftBeacon.longitude,
func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
AircraftBeacon.ground_speed,
AircraftBeacon.track,
func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
AircraftBeacon.ground_speed,
func.lag(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_prev'),
func.lead(AircraftBeacon.ground_speed).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('ground_speed_next'),
AircraftBeacon.altitude,
func.lag(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_prev'),
func.lead(AircraftBeacon.altitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('altitude_next')) \
.filter(AircraftBeacon.timestamp > last_takeoff_landing) \
.order_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.address, AircraftBeacon.timestamp) \
.subquery()
# find takeoffs and landings (look at the trigger_speed)
takeoff_landing_query = app.session.query(sq.c.address, sq.c.timestamp, sq.c.latitude, sq.c.longitude, sq.c.track, sq.c.ground_speed, sq.c.altitude, case([(sq.c.ground_speed>takeoff_speed, True), (sq.c.ground_speed<landing_speed, False)]).label('is_takeoff')) \
takeoff_landing_query = app.session.query(
sq.c.address,
sq.c.timestamp,
sq.c.latitude,
sq.c.longitude,
sq.c.track,
sq.c.ground_speed,
sq.c.altitude,
case([(sq.c.ground_speed > takeoff_speed, True),
(sq.c.ground_speed < landing_speed, False)]).label('is_takeoff')) \
.filter(sq.c.address_prev == sq.c.address == sq.c.address_next) \
.filter(or_(and_(sq.c.ground_speed_prev < takeoff_speed, # takeoff
sq.c.ground_speed > takeoff_speed,
@ -67,6 +77,3 @@ def compute_takeoff_and_landing():
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
app.session.execute(ins)
app.session.commit()

1
ogn/commands/database.py 100755 → 100644
Wyświetl plik

@ -5,6 +5,7 @@ manager = Manager()
from ogn.collect.fetchddb import update_ddb_data
@manager.command
def init():
"""Initialize the database."""

109
ogn/commands/logbook.py 100755 → 100644
Wyświetl plik

@ -1,18 +1,19 @@
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
from datetime import timedelta
from sqlalchemy.sql import func, null
from sqlalchemy import and_, or_, insert, between
from sqlalchemy.sql.expression import case, true, false, label
from ogn.model import Flarm, AircraftBeacon, TakeoffLanding
from ogn.model import Flarm, TakeoffLanding
from ogn.commands.dbutils import session
from manager import Manager
manager = Manager()
@manager.command
def show(airport_name, latitude, longitude, altitude):
"""Show a logbook for <airport_name> located at given position."""
@ -27,19 +28,62 @@ def show(airport_name, latitude, longitude, altitude):
max_altitude = altitude + 200
# make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
sq = session.query(TakeoffLanding.address,
func.lag(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_prev'),
func.lead(TakeoffLanding.address).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('address_next'),
TakeoffLanding.timestamp,
func.lag(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_prev'),
func.lead(TakeoffLanding.timestamp).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('timestamp_next'),
TakeoffLanding.track,
func.lag(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_prev'),
func.lead(TakeoffLanding.track).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('track_next'),
TakeoffLanding.is_takeoff,
func.lag(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_prev'),
func.lead(TakeoffLanding.is_takeoff).over(order_by=and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.address, TakeoffLanding.timestamp)).label('is_takeoff_next')) \
.filter(and_(between(TakeoffLanding.latitude, latmin, latmax), between(TakeoffLanding.longitude, lonmin, lonmax))) \
sq = session.query(
TakeoffLanding.address,
func.lag(TakeoffLanding.address) \
.over(
order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('address_prev'),
func.lead(TakeoffLanding.address) \
.over(
order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('address_next'),
TakeoffLanding.timestamp,
func.lag(
TakeoffLanding.timestamp) \
.over(
order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('timestamp_prev'),
func.lead(
TakeoffLanding.timestamp) \
.over(order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('timestamp_next'),
TakeoffLanding.track,
func.lag(
TakeoffLanding.track) \
.over(order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('track_prev'),
func.lead(
TakeoffLanding.track) \
.over(order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('track_next'),
TakeoffLanding.is_takeoff,
func.lag(
TakeoffLanding.is_takeoff) \
.over(order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('is_takeoff_prev'),
func.lead(
TakeoffLanding.is_takeoff) \
.over(order_by=and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.address,
TakeoffLanding.timestamp)) \
.label('is_takeoff_next')) \
.filter(and_(between(TakeoffLanding.latitude, latmin, latmax),
between(TakeoffLanding.longitude, lonmin, lonmax))) \
.filter(TakeoffLanding.altitude < max_altitude) \
.subquery()
@ -50,7 +94,7 @@ def show(airport_name, latitude, longitude, altitude):
.filter(sq.c.timestamp_next - sq.c.timestamp < timedelta(days=1))
# split complete flights (with takeoff and landing) with duration > 1 day into one takeoff and one landing
split_start_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'), null().label('duration')) \
split_start_query = session.query(sq.c.timestamp.label('reftime'), sq.c.address.label('address'), sq.c.timestamp.label('takeoff'), sq.c.track.label('takeoff_track'), null().label('landing'), null().label('landing_track'), null().label('duration')) \
.filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) \
.filter(sq.c.address == sq.c.address_next) \
.filter(sq.c.timestamp_next - sq.c.timestamp >= timedelta(days=1))
@ -73,19 +117,42 @@ def show(airport_name, latitude, longitude, altitude):
sq.c.is_takeoff_next == true()))
# unite all
union_query = complete_flight_query.union(split_start_query, split_landing_query, only_landings_query, only_starts_query) \
union_query = complete_flight_query.union(
split_start_query,
split_landing_query,
only_landings_query,
only_starts_query) \
.subquery()
# get aircraft informations and sort all entries by the reference time
logbook_query = session.query(union_query.c.reftime, union_query.c.address, union_query.c.takeoff, union_query.c.takeoff_track, union_query.c.landing, union_query.c.landing_track, union_query.c.duration, Flarm.registration, Flarm.aircraft) \
logbook_query = session.query(
union_query.c.reftime,
union_query.c.address,
union_query.c.takeoff,
union_query.c.takeoff_track,
union_query.c.landing,
union_query.c.landing_track,
union_query.c.duration,
Flarm.registration,
Flarm.aircraft) \
.outerjoin(Flarm, union_query.c.address == Flarm.address) \
.order_by(union_query.c.reftime)
print('--- Logbook (' + airport_name + ') ---')
print('--- Logbook (%s) ---' % airport_name)
none_datetime_replacer = lambda datetime_object: '--:--:--' if datetime_object is None else datetime_object.time()
none_track_replacer = lambda track_object: '--' if track_object is None else round(track_object/10.0)
none_track_replacer = lambda track_object: '--' if track_object is None else round(track_object / 10.0)
none_timedelta_replacer = lambda timedelta_object: '--:--:--' if timedelta_object is None else timedelta_object
none_registration_replacer = lambda registration_object, address: '[' + address + ']' if registration_object is None else registration_object
none_aircraft_replacer = lambda aircraft_object: '(unknown)' if aircraft_object is None else aircraft_object
for [reftime, address, takeoff, takeoff_track, landing, landing_track, duration, registration, aircraft] in logbook_query.all():
print('%10s %8s (%2s) %8s (%2s) %8s %8s %s' % (reftime.date(), none_datetime_replacer(takeoff), none_track_replacer(takeoff_track), none_datetime_replacer(landing), none_track_replacer(landing_track), none_timedelta_replacer(duration), none_registration_replacer(registration, address), none_aircraft_replacer(aircraft)))
print('%10s %8s (%2s) %8s (%2s) %8s %8s %s' % (
reftime.date(),
none_datetime_replacer(takeoff),
none_track_replacer(takeoff_track),
none_datetime_replacer(landing),
none_track_replacer(landing_track),
none_timedelta_replacer(duration),
none_registration_replacer(registration, address),
none_aircraft_replacer(aircraft)))

2
ogn/commands/showreceiver.py 100755 → 100644
Wyświetl plik

@ -9,7 +9,7 @@ from ogn.model.receiver_device import ReceiverDevice
from ogn.commands.dbutils import session
back_24h = datetime.utcnow() - timedelta(days=1)
receiver_messages_per_24h = 24*60 / 5
receiver_messages_per_24h = 24 * 60 / 5
from manager import Manager
manager = Manager()

18
ogn/model/aircraft_beacon.py
Wyświetl plik

@ -97,7 +97,7 @@ class AircraftBeacon(Beacon):
self.stealth = ((int(address_match.group(1), 16) & 0b10000000) >> 7 == 1)
self.address = address_match.group(2)
elif climb_rate_match is not None:
self.climb_rate = int(climb_rate_match.group(1))*fpm2ms
self.climb_rate = int(climb_rate_match.group(1)) * fpm2ms
elif turn_rate_match is not None:
self.turn_rate = float(turn_rate_match.group(1))
elif signal_strength_match is not None:
@ -130,4 +130,18 @@ class AircraftBeacon(Beacon):
raise Exception("No valid position description: %s" % part)
def __repr__(self):
return("<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.name, self.address_type, self.aircraft_type, self.timestamp, self.address_type, self.aircraft_type, self.stealth, self.address, self.climb_rate, self.turn_rate, self.signal_strength, self.error_count, self.frequency_offset, self.gps_status))
return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
self.name,
self.address_type,
self.aircraft_type,
self.timestamp,
self.address_type,
self.aircraft_type,
self.stealth,
self.address,
self.climb_rate,
self.turn_rate,
self.signal_strength,
self.error_count,
self.frequency_offset,
self.gps_status)

12
ogn/model/flarm.py
Wyświetl plik

@ -21,4 +21,14 @@ class Flarm(Base):
address_origin = Column(SmallInteger)
def __repr__(self):
return("<Flarm: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.address_type, self.address, self.name, self.airport, self.aircraft, self.registration, self.competition, self.frequency, self.tracked, self.identified))
return "<Flarm: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
self.address_type,
self.address,
self.name,
self.airport,
self.aircraft,
self.registration,
self.competition,
self.frequency,
self.tracked,
self.identified)

2
ogn/model/receiver_beacon.py
Wyświetl plik

@ -85,4 +85,4 @@ class ReceiverBeacon(Beacon):
raise Exception("No valid receiver description: %s" % part)
def __repr__(self):
return("<ReceiverBeacon %s: %s>" % (self.name, self.version))
return "<ReceiverBeacon %s: %s>" % (self.name, self.version)