mirror
/
ogn-python
kopia lustrzana https://github.com/glidernet/ogn-python
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Improvements and fixes

pull/68/head
Konstantin Gründger 2018-10-21 17:34:03 +02:00
rodzic 1d625d0f4f
commit 7c22840f04
26 zmienionych plików z 9913 dodań i 273 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
config/default.py
Wyświetl plik

@ -1,4 +1,4 @@
SQLALCHEMY_DATABASE_URI = 'postgresql:///ogn'
SQLALCHEMY_DATABASE_URI = 'postgresql://postgres@localhost:5432/ogn'
BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'

34
ogn/collect/database.py
Wyświetl plik

@ -6,18 +6,25 @@ from sqlalchemy.sql.expression import case
from ogn.collect.celery import app
from ogn.model import DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
from ogn.utils import get_ddb, get_country_code
from ogn.utils import get_ddb, get_country_code, get_flarmnet
logger = get_task_logger(__name__)
def update_device_infos(session, address_origin, csvfile=None):
device_infos = get_ddb(csvfile=csvfile, address_origin=address_origin)
def update_device_infos(session, address_origin, path=None):
if address_origin == DeviceInfoOrigin.flarmnet:
device_infos = get_flarmnet(fln_file=path)
else:
device_infos = get_ddb(csv_file=path)
session.query(DeviceInfo) \
.filter(DeviceInfo.address_origin == address_origin) \
.delete(synchronize_session='fetch')
session.commit()
for device_info in device_infos:
device_info.address_origin = address_origin
session.bulk_save_objects(device_infos)
session.commit()
@ -33,25 +40,7 @@ def import_ddb(session=None):
session = app.session
logger.info("Import registered devices fom the DDB...")
address_origin = DeviceInfoOrigin.ogn_ddb
counter = update_device_infos(session, address_origin)
logger.info("Imported {} devices.".format(counter))
return "Imported {} devices.".format(counter)
@app.task
def import_ddb_file(session=None, path='tests/custom_ddb.txt'):
"""Import registered devices from a local file."""
if session is None:
session = app.session
logger.info("Import registered devices from '{}'...".format(path))
address_origin = DeviceInfoOrigin.user_defined
counter = update_device_infos(session, address_origin, csvfile=path)
counter = update_device_infos(session, DeviceInfoOrigin.ogn_ddb)
logger.info("Imported {} devices.".format(counter))
return "Imported {} devices.".format(counter)
@ -150,6 +139,7 @@ def update_country_code(session=None):
for receiver in unknown_country_query.all():
location = receiver.location
country_code = get_country_code(location.latitude, location.longitude)
print("{}: {}".format(receiver.name, country_code))
if country_code is not None:
receiver.country_code = country_code
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))

8
ogn/collect/logbook.py
Wyświetl plik

@ -22,8 +22,12 @@ def update_logbook(session=None):
# 'wo' is the window order for the sql window function
wo = and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.device_id,
TakeoffLanding.timestamp,
TakeoffLanding.airport_id)
TakeoffLanding.airport_id,
TakeoffLanding.timestamp)
# 'pa' is the window partition for the sql window function
pa = (func.date(TakeoffLanding.timestamp),
TakeoffLanding.device_id)
# make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
sq = session.query(

372
ogn/collect/stats.py
Wyświetl plik

@ -6,7 +6,7 @@ from sqlalchemy import insert, distinct
from sqlalchemy.sql import null, and_, func, or_, update
from sqlalchemy.sql.expression import literal_column, case
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats, RelationStats, Receiver, Device
from .celery import app
from ogn.model.receiver_beacon import ReceiverBeacon
@ -15,7 +15,7 @@ logger = get_task_logger(__name__)
@app.task
def update_device_stats(session=None, date=None):
def create_device_stats(session=None, date=None):
"""Add/update device stats."""
if session is None:
@ -89,7 +89,7 @@ def update_device_stats(session=None, date=None):
@app.task
def update_receiver_stats(session=None, date=None):
def create_receiver_stats(session=None, date=None):
"""Add/update receiver stats."""
if session is None:
@ -104,30 +104,35 @@ def update_receiver_stats(session=None, date=None):
.filter(ReceiverStats.date == date) \
.delete()
# Calculate stats, firstseen, lastseen and last values != NULL
receiver_stats = session.query(
distinct(ReceiverBeacon.receiver_id).label('receiver_id'),
func.date(ReceiverBeacon.timestamp).label('date'),
func.first_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).asc().nullslast())
.label('firstseen'),
func.first_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.timestamp == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('lastseen'),
func.first_value(ReceiverBeacon.location_wkt)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.location_wkt == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('location_wkt'),
func.first_value(ReceiverBeacon.altitude)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.altitude == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('altitude'),
func.first_value(ReceiverBeacon.version)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.version == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('version'),
func.first_value(ReceiverBeacon.platform)
.over(partition_by=ReceiverBeacon.receiver_id, order_by=case([(ReceiverBeacon.platform == null(), None)], else_=ReceiverBeacon.timestamp).desc().nullslast())
.label('platform')) \
# Select one day
sq = session.query(ReceiverBeacon) \
.filter(func.date(ReceiverBeacon.timestamp) == date) \
.subquery()
# Calculate stats, firstseen, lastseen and last values != NULL
receiver_stats = session.query(
distinct(sq.c.receiver_id).label('receiver_id'),
func.date(sq.c.timestamp).label('date'),
func.first_value(sq.c.timestamp)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast())
.label('firstseen'),
func.first_value(sq.c.timestamp)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast())
.label('lastseen'),
func.first_value(sq.c.location)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.location == null(), None)], else_=sq.c.timestamp).desc().nullslast())
.label('location_wkt'),
func.first_value(sq.c.altitude)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.altitude == null(), None)], else_=sq.c.timestamp).desc().nullslast())
.label('altitude'),
func.first_value(sq.c.version)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.version == null(), None)], else_=sq.c.timestamp).desc().nullslast())
.label('version'),
func.first_value(sq.c.platform)
.over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.platform == null(), None)], else_=sq.c.timestamp).desc().nullslast())
.label('platform')) \
.subquery()
# And insert them
ins = insert(ReceiverStats).from_select(
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.firstseen, ReceiverStats.lastseen, ReceiverStats.location_wkt, ReceiverStats.altitude, ReceiverStats.version, ReceiverStats.platform],
@ -137,16 +142,315 @@ def update_receiver_stats(session=None, date=None):
session.commit()
logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
# Update AircraftBeacon distances
upd = update(AircraftBeacon) \
.where(and_(func.date(AircraftBeacon.timestamp) == ReceiverStats.date,
AircraftBeacon.receiver_id == ReceiverStats.receiver_id,
AircraftBeacon.distance == null())) \
.values({"distance": func.ST_Distance_Sphere(AircraftBeacon.location_wkt, ReceiverStats.location_wkt)})
# Update aircraft_beacon_count, aircraft_count and max_distance (without any error and max quality of 36dB@10km which is enough for 640km ... )
aircraft_beacon_stats = session.query(func.date(AircraftBeacon.timestamp).label('date'),
AircraftBeacon.receiver_id,
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
func.count(func.distinct(AircraftBeacon.device_id)).label('aircraft_count'),
func.max(AircraftBeacon.distance).label('max_distance')) \
.filter(and_(func.date(AircraftBeacon.timestamp) == date,
AircraftBeacon.error_count == 0,
AircraftBeacon.quality <= 40)) \
.group_by(func.date(AircraftBeacon.timestamp),
AircraftBeacon.receiver_id) \
.subquery()
upd = update(ReceiverStats) \
.where(and_(ReceiverStats.date == aircraft_beacon_stats.c.date,
ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id)) \
.values({'aircraft_beacon_count': aircraft_beacon_stats.c.aircraft_beacon_count,
'aircraft_count': aircraft_beacon_stats.c.aircraft_count,
'max_distance': aircraft_beacon_stats.c.max_distance})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} AircraftBeacons".format(update_counter))
logger.warn("Updated {} ReceiverStats".format(update_counter))
return "ReceiverStats for {}: {} deleted, {} inserted, AircraftBeacons: {} updated".format(date, deleted_counter, insert_counter, update_counter)
return "ReceiverStats for {}: {} deleted, {} inserted, {} updated".format(date, deleted_counter, insert_counter, update_counter)
@app.task
def update_device_stats_jumps(session=None, date=None):
"""Update device stats jumps."""
if session is None:
session = app.session
if not date:
logger.warn("A date is needed for calculating device stats jumps. Exiting")
return None
sq = session.query(AircraftBeacon.device_id,
AircraftBeacon.timestamp.label('t0'),
func.lead(AircraftBeacon.timestamp).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('t1'),
AircraftBeacon.location_wkt.label('l0'),
func.lead(AircraftBeacon.location_wkt).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('l1'),
AircraftBeacon.altitude.label('a0'),
func.lead(AircraftBeacon.altitude).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label('a1')) \
.filter(and_(func.date(AircraftBeacon.timestamp) == date,
AircraftBeacon.error_count == 0)) \
.subquery()
sq2 = session.query(sq.c.device_id,
(func.st_distancesphere(sq.c.l1, sq.c.l0) / (func.extract('epoch', sq.c.t1) - func.extract('epoch', sq.c.t0))).label('horizontal_speed'),
((sq.c.a1 - sq.c.a0) / (func.extract('epoch', sq.c.t1) - func.extract('epoch', sq.c.t0))).label('vertical_speed')) \
.filter(and_(sq.c.t0 != null(),
sq.c.t1 != null(),
sq.c.t0 < sq.c.t1)) \
.subquery()
sq3 = session.query(sq2.c.device_id,
case([(or_(func.abs(sq2.c.horizontal_speed) > 1000, func.abs(sq2.c.vertical_speed) > 100), 1)], else_=0).label('jump')) \
.subquery()
sq4 = session.query(sq3.c.device_id,
func.sum(sq3.c.jump).label('jumps')) \
.group_by(sq3.c.device_id) \
.subquery()
upd = update(DeviceStats) \
.where(and_(DeviceStats.date == date,
DeviceStats.device_id == sq4.c.device_id)) \
.values({'ambiguous': sq4.c.jumps > 10,
'jumps': sq4.c.jumps})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} DeviceStats jumps".format(update_counter))
return "DeviceStats jumps for {}: {} updated".format(date, update_counter)
@app.task
def create_relation_stats(session=None, date=None):
"""Add/update relation stats."""
if session is None:
session = app.session
if not date:
logger.warn("A date is needed for calculating stats. Exiting")
return None
# First kill the stats for the selected date
deleted_counter = session.query(RelationStats) \
.filter(RelationStats.date == date) \
.delete()
# Calculate stats for selected day
relation_stats = session.query(
func.date(AircraftBeacon.timestamp),
AircraftBeacon.device_id,
AircraftBeacon.receiver_id,
func.max(AircraftBeacon.quality),
func.count(AircraftBeacon.id)
) \
.filter(and_(func.date(AircraftBeacon.timestamp) == date,
AircraftBeacon.distance > 1000,
AircraftBeacon.error_count == 0,
AircraftBeacon.quality <= 40,
AircraftBeacon.ground_speed > 10)) \
.group_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.receiver_id) \
.subquery()
# And insert them
ins = insert(RelationStats).from_select(
[RelationStats.date, RelationStats.device_id, RelationStats.receiver_id, RelationStats.quality, RelationStats.beacon_count],
relation_stats)
res = session.execute(ins)
insert_counter = res.rowcount
session.commit()
logger.warn("RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
return "RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
@app.task
def update_qualities(session=None, date=None):
"""Calculate relative qualities of receivers and devices."""
if session is None:
session = app.session
if not date:
logger.warn("A date is needed for update stats. Exiting")
return None
# Calculate avg quality of devices
dev_sq = session.query(RelationStats.date,
RelationStats.device_id,
func.avg(RelationStats.quality).label('quality')) \
.filter(RelationStats.date == date) \
.group_by(RelationStats.date,
RelationStats.device_id) \
.subquery()
dev_upd = update(DeviceStats) \
.where(and_(DeviceStats.date == dev_sq.c.date,
DeviceStats.device_id == dev_sq.c.device_id)) \
.values({'quality': dev_sq.c.quality})
dev_result = session.execute(dev_upd)
dev_update_counter = dev_result.rowcount
session.commit()
logger.warn("Updated {} DeviceStats: quality".format(dev_update_counter))
# Calculate avg quality of receivers
rec_sq = session.query(RelationStats.date,
RelationStats.receiver_id,
func.avg(RelationStats.quality).label('quality')) \
.filter(RelationStats.date == date) \
.group_by(RelationStats.date,
RelationStats.receiver_id) \
.subquery()
rec_upd = update(ReceiverStats) \
.where(and_(ReceiverStats.date == rec_sq.c.date,
ReceiverStats.receiver_id == rec_sq.c.receiver_id)) \
.values({'quality': rec_sq.c.quality})
rec_result = session.execute(rec_upd)
rec_update_counter = rec_result.rowcount
session.commit()
logger.warn("Updated {} ReceiverStats: quality".format(rec_update_counter))
# Calculate quality_offset of devices
dev_sq = session.query(RelationStats.date,
RelationStats.device_id,
(func.sum(RelationStats.beacon_count*(RelationStats.quality - ReceiverStats.quality))/(func.sum(RelationStats.beacon_count))).label('quality_offset')) \
.filter(RelationStats.date == date) \
.filter(and_(RelationStats.receiver_id == ReceiverStats.receiver_id,
RelationStats.date == ReceiverStats.date)) \
.group_by(RelationStats.date,
RelationStats.device_id) \
.subquery()
dev_upd = update(DeviceStats) \
.where(and_(DeviceStats.date == dev_sq.c.date,
DeviceStats.device_id == dev_sq.c.device_id)) \
.values({'quality_offset': dev_sq.c.quality_offset})
dev_result = session.execute(dev_upd)
dev_update_counter = dev_result.rowcount
session.commit()
logger.warn("Updated {} DeviceStats: quality_offset".format(dev_update_counter))
# Calculate quality_offset of receivers
rec_sq = session.query(RelationStats.date,
RelationStats.receiver_id,
(func.sum(RelationStats.beacon_count*(RelationStats.quality - DeviceStats.quality))/(func.sum(RelationStats.beacon_count))).label('quality_offset')) \
.filter(RelationStats.date == date) \
.filter(and_(RelationStats.device_id == DeviceStats.device_id,
RelationStats.date == DeviceStats.date)) \
.group_by(RelationStats.date,
RelationStats.receiver_id) \
.subquery()
rec_upd = update(ReceiverStats) \
.where(and_(ReceiverStats.date == rec_sq.c.date,
ReceiverStats.receiver_id == rec_sq.c.receiver_id)) \
.values({'quality_offset': rec_sq.c.quality_offset})
rec_result = session.execute(rec_upd)
rec_update_counter = rec_result.rowcount
session.commit()
logger.warn("Updated {} ReceiverStats: quality_offset".format(rec_update_counter))
return "Updated {} DeviceStats and {} ReceiverStats".format(dev_update_counter, rec_update_counter)
@app.task
def update_receivers_stats(session=None):
"""Update receivers with stats."""
if session is None:
session = app.session
receiver_stats = session.query(
distinct(ReceiverStats.receiver_id).label('receiver_id'),
func.first_value(ReceiverStats.firstseen)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.firstseen == null(), None)], else_=ReceiverStats.date).asc().nullslast())
.label('firstseen'),
func.first_value(ReceiverStats.lastseen)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.lastseen == null(), None)], else_=ReceiverStats.date).desc().nullslast())
.label('lastseen'),
func.first_value(ReceiverStats.location_wkt)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.location_wkt == null(), None)], else_=ReceiverStats.date).desc().nullslast())
.label('location_wkt'),
func.first_value(ReceiverStats.altitude)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.altitude == null(), None)], else_=ReceiverStats.date).desc().nullslast())
.label('altitude'),
func.first_value(ReceiverStats.version)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.version == null(), None)], else_=ReceiverStats.date).desc().nullslast())
.label('version'),
func.first_value(ReceiverStats.platform)
.over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.platform == null(), None)], else_=ReceiverStats.date).desc().nullslast())
.label('platform')) \
.order_by(ReceiverStats.receiver_id) \
.subquery()
upd = update(Receiver) \
.where(and_(Receiver.id == receiver_stats.c.receiver_id)) \
.values({'firstseen': receiver_stats.c.firstseen,
'lastseen': receiver_stats.c.lastseen,
'location': receiver_stats.c.location_wkt,
'altitude': receiver_stats.c.altitude,
'version': receiver_stats.c.version,
'platform': receiver_stats.c.platform})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} Receivers".format(update_counter))
return "Updated {} Receivers".format(update_counter)
@app.task
def update_devices_stats(session=None):
"""Update devices with stats."""
if session is None:
session = app.session
device_stats = session.query(
distinct(DeviceStats.device_id).label('device_id'),
func.first_value(DeviceStats.firstseen)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.firstseen == null(), None)], else_=DeviceStats.date).asc().nullslast())
.label('firstseen'),
func.max(DeviceStats.lastseen)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.lastseen == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('lastseen'),
func.first_value(DeviceStats.aircraft_type)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.aircraft_type == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('aircraft_type'),
func.first_value(DeviceStats.stealth)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.stealth == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('stealth'),
func.first_value(DeviceStats.software_version)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.software_version == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('software_version'),
func.first_value(DeviceStats.hardware_version)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.hardware_version == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('hardware_version'),
func.first_value(DeviceStats.real_address)
.over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.real_address == null(), None)], else_=DeviceStats.date).desc().nullslast())
.label('real_address')) \
.order_by(DeviceStats.device_id) \
.subquery()
upd = update(Device) \
.where(and_(Device.id == device_stats.c.device_id)) \
.values({'firstseen': device_stats.c.firstseen,
'lastseen': device_stats.c.lastseen,
'aircraft_type': device_stats.c.aircraft_type,
'stealth': device_stats.c.stealth,
'software_version': device_stats.c.software_version,
'hardware_version': device_stats.c.hardware_version,
'real_address': device_stats.c.real_address})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} Devices".format(update_counter))
return "Updated {} Devices".format(update_counter)

139
ogn/collect/takeoff_landings.py
Wyświetl plik

@ -15,14 +15,14 @@ logger = get_task_logger(__name__)
@app.task
def update_takeoff_landings(session=None, date=None):
"""Compute takeoffs and landings."""
logger.info("Compute takeoffs and landings.")
if session is None:
session = app.session
# check if we have any airport
airports_query = session.query(Airport)
airports_query = session.query(Airport).limit(1)
if not airports_query.all():
logger.warn("Cannot calculate takeoff and landings without any airport! Please import airports first.")
return
@ -38,30 +38,25 @@ def update_takeoff_landings(session=None, date=None):
airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport
# 'wo' is the window order for the sql window function
wo = and_(AircraftBeacon.device_id,
AircraftBeacon.timestamp,
AircraftBeacon.receiver_id)
wo = and_(func.date(AircraftBeacon.timestamp),
AircraftBeacon.device_id,
AircraftBeacon.timestamp)
# get beacons for selected day and filter out duplicates (e.g. from multiple receivers)
sq = session.query(AircraftBeacon.id,
func.row_number().over(partition_by=(func.date(AircraftBeacon.timestamp),
AircraftBeacon.device_id,
AircraftBeacon.timestamp),
order_by=AircraftBeacon.error_count).label('row')) \
.filter(func.date(AircraftBeacon.timestamp) == date) \
.subquery()
sq2 = session.query(sq.c.id) \
.filter(sq.c.row == 1) \
.subquery()
# make a query with current, previous and next position
if date is None:
beacon_selection = session.query(AircraftBeacon.id) \
.filter(AircraftBeacon.status == 0) \
.order_by(AircraftBeacon.timestamp) \
.subquery()
else:
my_day = datetime.strptime(date, '%Y-%m-%d')
beacon_selection = session.query(AircraftBeacon.id) \
.filter(and_(AircraftBeacon.status == 0,
AircraftBeacon.timestamp >= my_day - timedelta(minutes=5),
AircraftBeacon.timestamp < my_day + timedelta(days=1, minutes=5))) \
.order_by(AircraftBeacon.timestamp) \
.limit(100000) \
.subquery()
sq = session.query(
AircraftBeacon.id,
func.lag(AircraftBeacon.id).over(order_by=wo).label('id_prev'),
func.lead(AircraftBeacon.id).over(order_by=wo).label('id_next'),
sq3 = session.query(
AircraftBeacon.device_id,
func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'),
func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next'),
@ -80,60 +75,56 @@ def update_takeoff_landings(session=None, date=None):
AircraftBeacon.altitude,
func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'),
func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next')) \
.filter(AircraftBeacon.id == beacon_selection.c.id) \
.filter(AircraftBeacon.id == sq2.c.id) \
.subquery()
# consider only positions with the same device id
sq2 = session.query(sq) \
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
sq4 = session.query(sq3) \
.filter(sq3.c.device_id_prev == sq3.c.device_id == sq3.c.device_id_next) \
.subquery()
logger.warn(sq2)
return
# find possible takeoffs and landings
sq3 = session.query(
sq2.c.id,
sq2.c.timestamp,
case([(sq2.c.ground_speed > takeoff_speed, sq2.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport
(sq2.c.ground_speed < landing_speed, sq2.c.location)]).label('location'),
case([(sq2.c.ground_speed > takeoff_speed, sq2.c.track),
(sq2.c.ground_speed < landing_speed, sq2.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly
sq2.c.ground_speed,
sq2.c.altitude,
case([(sq2.c.ground_speed > takeoff_speed, True),
(sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'),
sq2.c.device_id) \
.filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \
.filter(and_(func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_prev) < radius,
func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_next) < radius)) \
.filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff
sq2.c.ground_speed > takeoff_speed,
sq2.c.ground_speed_next > takeoff_speed),
and_(sq2.c.ground_speed_prev > landing_speed, # landing
sq2.c.ground_speed < landing_speed,
sq2.c.ground_speed_next < landing_speed))) \
sq5 = session.query(
sq4.c.timestamp,
case([(sq4.c.ground_speed > takeoff_speed, sq4.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport
(sq4.c.ground_speed < landing_speed, sq4.c.location)]).label('location'),
case([(sq4.c.ground_speed > takeoff_speed, sq4.c.track),
(sq4.c.ground_speed < landing_speed, sq4.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly
sq4.c.ground_speed,
sq4.c.altitude,
case([(sq4.c.ground_speed > takeoff_speed, True),
(sq4.c.ground_speed < landing_speed, False)]).label('is_takeoff'),
sq4.c.device_id) \
.filter(sq4.c.timestamp_next - sq4.c.timestamp_prev < timedelta(seconds=duration)) \
.filter(and_(func.ST_DistanceSphere(sq4.c.location, sq4.c.location_wkt_prev) < radius,
func.ST_DistanceSphere(sq4.c.location, sq4.c.location_wkt_next) < radius)) \
.filter(or_(and_(sq4.c.ground_speed_prev < takeoff_speed, # takeoff
sq4.c.ground_speed > takeoff_speed,
sq4.c.ground_speed_next > takeoff_speed),
and_(sq4.c.ground_speed_prev > landing_speed, # landing
sq4.c.ground_speed < landing_speed,
sq4.c.ground_speed_next < landing_speed))) \
.subquery()
# consider them if they are near a airport
sq4 = session.query(
sq3.c.timestamp,
sq3.c.track,
sq3.c.is_takeoff,
sq3.c.device_id,
sq6 = session.query(
sq5.c.timestamp,
sq5.c.track,
sq5.c.is_takeoff,
sq5.c.device_id,
Airport.id.label('airport_id')) \
.filter(and_(func.ST_Distance_Sphere(sq3.c.location, Airport.location_wkt) < airport_radius,
between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \
.filter(and_(func.ST_DistanceSphere(sq5.c.location, Airport.location_wkt) < airport_radius,
between(sq5.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \
.filter(between(Airport.style, 2, 5)) \
.subquery()
# consider them only if they are not already existing in db
takeoff_landing_query = session.query(sq4) \
takeoff_landing_query = session.query(sq6) \
.filter(~exists().where(
and_(TakeoffLanding.timestamp == sq4.c.timestamp,
TakeoffLanding.device_id == sq4.c.device_id,
TakeoffLanding.airport_id == sq4.c.airport_id)))
and_(TakeoffLanding.timestamp == sq6.c.timestamp,
TakeoffLanding.device_id == sq6.c.device_id,
TakeoffLanding.airport_id == sq6.c.airport_id)))
# ... and save them
ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp,
TakeoffLanding.track,
@ -141,18 +132,10 @@ def update_takeoff_landings(session=None, date=None):
TakeoffLanding.device_id,
TakeoffLanding.airport_id),
takeoff_landing_query)
result = session.execute(ins)
session.commit()
insert_counter = result.rowcount
logger.warn("Inserted {} TakeoffLandings".format(insert_counter))
# Set calculated beacons as 'used'
upd = update(AircraftBeacon) \
.where(AircraftBeacon.id == sq2.c.id) \
.values({"status": 1})
result = session.execute(upd)
update_counter = result.rowcount
session.commit()
logger.warn("Updated {} AircraftBeacons".format(update_counter))
return "Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(insert_counter, update_counter)
return "Inserted {} TakeoffLandings".format(insert_counter)

2
ogn/commands/__init__.py
Wyświetl plik

@ -6,6 +6,7 @@ from .showreceiver import manager as show_receiver_manager
from .showdevices import manager as show_devices_manager
from .showdeviceinfos import manager as show_deviceinfos_manager
from .logbook import manager as logbook_manager
from ogn.commands.stats import manager as stats_manager
from manager import Manager
@ -19,3 +20,4 @@ manager.merge(show_receiver_manager, namespace='show.receiver')
manager.merge(show_devices_manager, namespace='show.devices')
manager.merge(show_deviceinfos_manager, namespace='show.deviceinfos')
manager.merge(logbook_manager, namespace='logbook')
manager.merge(stats_manager, namespace='stats')

47
ogn/commands/bulkimport.py
Wyświetl plik

@ -25,7 +25,7 @@ def convert_logfile(path):
convert(tail, path=head)
logging.info("Finished converting single file {}".format(head))
elif os.path.isdir(path):
for filename in os.listdir(path):
for filename in sorted(os.listdir(path)):
convert(filename, path=path)
logging.info("Finished converting file path {}".format(path))
else:
@ -60,10 +60,14 @@ def convert(sourcefile, path=''):
fin = open_file(os.path.join(path, sourcefile))
# get total lines of the input file
total_lines = 0
for line in fin:
total_lines += 1
fin.seek(0)
try:
total_lines = 0
for line in fin: # Der Log vom 3.4.2018 und 24.6.2018 und 25.06.2018 und 24.07.2018 geht hier krachen
total_lines += 1
fin.seek(0)
except Exception as e:
print(e)
return
progress = -1
current_line = 0
@ -80,8 +84,11 @@ def convert(sourcefile, path=''):
print("=====")
print(line.strip())
continue
merger.add_message(message)
try:
merger.add_message(message)
except Exception as e:
print(e)
merger.flush()
saver.close()
@ -148,7 +155,7 @@ def import_csv_logfile(path, logfile='main.log', loglevel='INFO'):
import_logfile(path)
elif os.path.isdir(path):
print("{}: Scanning path: {}".format(datetime.datetime.now(), path))
for filename in os.listdir(path):
for filename in sorted(os.listdir(path)):
print("{}: Importing file: {}".format(datetime.datetime.now(), filename))
import_logfile(os.path.join(path, filename))
else:
@ -177,15 +184,9 @@ def import_logfile(path):
receiver_beacon_header = ','.join(ReceiverBeacon.get_csv_columns())
if header == aircraft_beacon_header:
if check_no_beacons('aircraft_beacons', reference_date_string):
import_aircraft_beacon_logfile(path)
else:
print("For {} beacons already exist. Skipping".format(reference_date_string))
import_aircraft_beacon_logfile(path)
elif header == receiver_beacon_header:
if check_no_beacons('receiver_beacons', reference_date_string):
import_receiver_beacon_logfile(path)
else:
print("For {} beacons already exist. Skipping".format(reference_date_string))
import_receiver_beacon_logfile(path)
else:
s1 = header
s2 = ','.join(AircraftBeacon.get_csv_columns())
@ -195,14 +196,6 @@ def import_logfile(path):
print("Unknown file type: {}".format(tail))
def check_no_beacons(tablename, reference_date_string):
result = session.execute("""SELECT * FROM {0} WHERE timestamp BETWEEN '{1} 00:00:00' AND '{1} 23:59:59' LIMIT 1""".format(tablename, reference_date_string))
if result.fetchall():
return False
else:
return True
def import_aircraft_beacon_logfile(csv_logfile):
SQL_TEMPTABLE_STATEMENT = """
DROP TABLE IF EXISTS aircraft_beacons_temp;
@ -235,7 +228,7 @@ def import_aircraft_beacon_logfile(csv_logfile):
distance real,
radial smallint,
normalized_signal_quality real,
quality real,
location_mgrs character varying(15)
);
"""
@ -284,11 +277,11 @@ def import_aircraft_beacon_logfile(csv_logfile):
session.execute("""
INSERT INTO aircraft_beacons(location, altitude, name, dstcall, relay, receiver_name, timestamp, track, ground_speed,
address_type, aircraft_type, stealth, address, climb_rate, turn_rate, signal_quality, error_count, frequency_offset, gps_quality_horizontal, gps_quality_vertical, software_version, hardware_version, real_address, signal_power,
distance, radial, normalized_signal_quality, location_mgrs,
distance, radial, quality, location_mgrs,
receiver_id, device_id)
SELECT t.location, t.altitude, t.name, t.dstcall, t.relay, t.receiver_name, t.timestamp, t.track, t.ground_speed,
t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.signal_quality, t.error_count, t.frequency_offset, t.gps_quality_horizontal, t.gps_quality_vertical, t.software_version, t.hardware_version, t.real_address, t.signal_power,
t.distance, t.radial, t.normalized_signal_quality, t.location_mgrs,
t.distance, t.radial, t.quality, t.location_mgrs,
r.id, d.id
FROM aircraft_beacons_temp t, receivers r, devices d
WHERE t.receiver_name = r.name AND t.address = d.address

30
ogn/commands/database.py
Wyświetl plik

@ -1,7 +1,7 @@
from manager import Manager
from ogn.collect.database import update_device_infos
from ogn.collect.database import update_device_infos, update_country_code
from ogn.commands.dbutils import engine, session
from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon
from ogn.model import Base, DeviceInfoOrigin
from ogn.utils import get_airports
from sqlalchemy import distinct
from sqlalchemy.sql import null, func
@ -20,6 +20,7 @@ def init():
from alembic import command
session.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
session.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;')
session.commit()
Base.metadata.create_all(engine)
#alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
@ -53,24 +54,29 @@ def import_ddb():
"""Import registered devices from the DDB."""
print("Import registered devices fom the DDB...")
address_origin = DeviceInfoOrigin.ogn_ddb
counter = update_device_infos(session,
address_origin)
counter = update_device_infos(session, DeviceInfoOrigin.ogn_ddb)
print("Imported %i devices." % counter)
@manager.command
def import_file(path='tests/custom_ddb.txt'):
"""Import registered devices from a local file."""
# (flushes previously manually imported entries)
print("Import registered devices from '{}'...".format(path))
address_origin = DeviceInfoOrigin.user_defined
counter = update_device_infos(session,
address_origin,
csvfile=path)
DeviceInfoOrigin.user_defined,
path=path)
print("Imported %i devices." % counter)
@manager.command
def import_flarmnet(path='tests/data.fln'):
"""Import registered devices from a local file."""
print("Import registered devices from '{}'...".format("internet" if path is None else path))
counter = update_device_infos(session,
DeviceInfoOrigin.flarmnet,
path=path)
print("Imported %i devices." % counter)
@manager.command
def import_airports(path='tests/SeeYou.cup'):
@ -81,3 +87,9 @@ def import_airports(path='tests/SeeYou.cup'):
session.bulk_save_objects(airports)
session.commit()
print("Imported {} airports.".format(len(airports)))
@manager.command
def update_country_codes():
"""Update country codes of all receivers."""
update_country_code(session=session)

79
ogn/commands/logbook.py
Wyświetl plik

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
from datetime import timedelta, datetime
from datetime import timedelta, datetime, date
from manager import Manager
from ogn.collect.logbook import update_logbook
@ -18,24 +18,24 @@ manager = Manager()
@manager.command
def compute_takeoff_landing():
"""Compute takeoffs and landings."""
print("Compute takeoffs and landings...")
result = update_takeoff_landings.delay()
counter = result.get()
print("New takeoffs/landings: {}".format(counter))
for single_date in (date(2017, 6, 14) + timedelta(days=n) for n in range(800)):
print("Berechne für den {}".format(single_date.strftime("%Y-%m-%d")))
result = update_takeoff_landings(session=session, date=single_date)
print(result)
@manager.command
def compute_logbook():
"""Compute logbook."""
print("Compute logbook...")
result = update_logbook.delay()
counter = result.get()
print("New logbook entries: {}".format(counter))
result = update_logbook(session=session)#.delay()
#counter = result.get()
#print("New logbook entries: {}".format(counter))
@manager.arg('date', help='date (format: yyyy-mm-dd)')
@manager.command
def show(airport_name, utc_delta_hours=0, date=None):
def show(airport_name, date=None):
"""Show a logbook for <airport_name>."""
airport = session.query(Airport) \
.filter(Airport.name == airport_name) \
@ -48,33 +48,14 @@ def show(airport_name, utc_delta_hours=0, date=None):
or_args = []
if date is not None:
date = datetime.strptime(date, "%Y-%m-%d")
or_args = [and_(TakeoffLanding.timestamp >= date,
TakeoffLanding.timestamp < date + timedelta(hours=24))]
# get device info with highes priority
sq2 = session.query(DeviceInfo.address, func.max(DeviceInfo.address_origin).label('address_origin')) \
.group_by(DeviceInfo.address) \
.subquery()
sq3 = session.query(DeviceInfo.address, DeviceInfo.registration, DeviceInfo.aircraft) \
.filter(and_(DeviceInfo.address == sq2.c.address, DeviceInfo.address_origin == sq2.c.address_origin)) \
.subquery()
or_args = [func.date(Logbook.reftime) == date]
# get all logbook entries and add device and airport infos
takeoff_airport = aliased(Airport, name='takeoff_airport')
landing_airport = aliased(Airport, name='landing_airport')
logbook_query = session.query(func.row_number().over(order_by=Logbook.reftime).label('row_number'),
Logbook,
Device,
sq3.c.registration,
sq3.c.aircraft) \
Logbook) \
.filter(*or_args) \
.filter(or_(Logbook.takeoff_airport_id == airport.id,
Logbook.landing_airport_id == airport.id)) \
.filter(*or_args) \
.outerjoin(takeoff_airport, Logbook.takeoff_airport_id == takeoff_airport.id) \
.outerjoin(landing_airport, Logbook.landing_airport_id == landing_airport.id) \
.outerjoin(Device, Logbook.device_id == Device.id) \
.outerjoin(sq3, sq3.c.address == Device.address) \
.order_by(Logbook.reftime)
# ... and finally print out the logbook
@ -89,24 +70,24 @@ def show(airport_name, utc_delta_hours=0, date=None):
def none_timedelta_replacer(timedelta_object):
return '--:--:--' if timedelta_object is None else timedelta_object
def none_registration_replacer(device_object, registration_object):
return '[' + device_object.address + ']' if registration_object is None else registration_object
def none_registration_replacer(device_object):
return '[' + device_object.address + ']' if len(device_object.infos) == 0 else device_object.infos[0].registration
def none_aircraft_replacer(device_object, aircraft_object):
return '(unknown)' if aircraft_object is None else aircraft_object
def none_aircraft_replacer(device_object):
return '(unknown)' if len(device_object.infos) == 0 else device_object.infos[0].aircraft
def airport_marker(takeoff_airport_object, landing_airport_object):
if takeoff_airport_object is not None and takeoff_airport_object.name is not airport.name:
return ('FROM: {}'.format(takeoff_airport_object.name))
elif landing_airport_object is not None and landing_airport_object.name is not airport.name:
return ('TO: {}'.format(landing_airport_object.name))
def airport_marker(logbook_object):
if logbook_object.takeoff_airport is not None and logbook_object.takeoff_airport.name is not airport.name:
return ('FROM: {}'.format(logbook_object.takeoff_airport.name))
elif logbook_object.landing_airport is not None and logbook_object.landing_airport.name is not airport.name:
return ('TO: {}'.format(logbook_object.landing_airport.name))
else:
return ('')
def none_altitude_replacer(altitude_object, airport_object):
return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude)
def none_altitude_replacer(logbook_object):
return "?" if logbook_object.max_altitude is None else "{:5d}m ({:+5d}m)".format(logbook_object.max_altitude, logbook_object.max_altitude - logbook_object.takeoff_airport.altitude)
for [row_number, logbook, device, registration, aircraft] in logbook_query.all():
for [row_number, logbook] in logbook_query.all():
print('%3d. %10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % (
row_number,
logbook.reftime.date(),
@ -115,7 +96,7 @@ def show(airport_name, utc_delta_hours=0, date=None):
none_datetime_replacer(logbook.landing_timestamp),
none_track_replacer(logbook.landing_track),
none_timedelta_replacer(logbook.duration),
none_altitude_replacer(logbook.max_altitude, logbook.takeoff_airport),
none_registration_replacer(device, registration),
none_aircraft_replacer(device, aircraft),
airport_marker(logbook.takeoff_airport, logbook.landing_airport)))
none_altitude_replacer(logbook),
none_registration_replacer(logbook.device),
none_aircraft_replacer(logbook.device),
airport_marker(logbook)))

114
ogn/commands/stats.py 100644
Wyświetl plik

@ -0,0 +1,114 @@
from manager import Manager
from ogn.commands.dbutils import session
from datetime import date, timedelta
from ogn.collect.stats import create_device_stats, create_receiver_stats, create_relation_stats,\
update_qualities, update_receivers_stats, update_devices_stats,\
update_device_stats_jumps
manager = Manager()
@manager.command
def create_stats():
"""Create DeviceStats, ReceiverStats and RelationStats."""
for single_date in (date(2016, 1, 1) + timedelta(days=n) for n in range(800)):
result = create_device_stats(session=session, date=single_date)
print(result)
result = update_device_stats_jumps(session=session, date=single_date)
print(result)
result = create_receiver_stats(session=session, date=single_date)
print(result)
result = create_relation_stats(session=session, date=single_date)
print(result)
result = update_qualities(session=session, date=single_date)
print(result)
#result = update_device_stats(session=session, date=date)
#print(result)
@manager.command
def update_receivers():
"""Update receivers with data from stats."""
result = update_receivers_stats(session=session)
print(result)
@manager.command
def update_devices():
"""Update devices with data from stats."""
result = update_devices_stats(session=session)
print(result)
@manager.command
def create_flights():
"""Create Flights."""
for single_date in (date(2016, 8, 10) + timedelta(days=n) for n in range(800)):
result = _create_flights2d(session=session, date=single_date)
#result = _create_flights3d(session=session, date=single_date)
print(result)
def _create_flights2d(session=None, date=None):
SQL = """
INSERT INTO flights2d
(
date,
device_id,
path
)
SELECT sq5.date,
sq5.device_id,
st_collect(sq5.linestring order BY sq5.part) multilinestring
FROM (
SELECT sq4.timestamp::date AS date,
sq4.device_id,
sq4.part,
st_makeline(sq4.location ORDER BY sq4.timestamp) linestring
FROM (
SELECT sq3.timestamp,
sq3.location,
sq3.device_id,
sum(sq3.ping) OVER (partition BY sq3.timestamp::date, sq3.device_id ORDER BY sq3.timestamp) part
FROM (
SELECT sq2.t1 AS timestamp,
sq2.l1 AS location,
sq2.d1 device_id,
CASE
WHEN sq2.t1 - sq2.t2 < interval'100s'
AND st_distancesphere(sq2.l1, sq2.l2) < 1000 THEN 0
ELSE 1
END AS ping
FROM (
SELECT sq.timestamp t1,
lag(sq.timestamp) OVER (partition BY sq.timestamp::date, sq.device_id ORDER BY sq.timestamp) t2,
sq.location l1,
lag(sq.location) OVER (partition BY sq.timestamp::date, sq.device_id ORDER BY sq.timestamp) l2,
sq.device_id d1,
lag(sq.device_id) OVER (partition BY sq.timestamp::date, sq.device_id ORDER BY sq.timestamp) d2
FROM (
SELECT timestamp,
device_id,
location,
row_number() OVER (partition BY timestamp::date, device_id, timestamp ORDER BY error_count) message_number
FROM aircraft_beacons
WHERE timestamp::date = '{}' ) sq
WHERE sq.message_number = 1 ) sq2 ) sq3 ) sq4
GROUP BY sq4.timestamp::date,
sq4.device_id,
sq4.part ) sq5
GROUP BY sq5.date,
sq5.device_id
"""
result = session.execute(SQL.format(date.strftime("%Y-%m-%d")))
insert_counter = result.rowcount
session.commit()
return "Inserted {} Flights for {}".format(insert_counter, date.strftime("%Y-%m-%d"))

6
ogn/gateway/manage.py
Wyświetl plik

@ -4,8 +4,8 @@ from manager import Manager
from ogn.client import AprsClient
from ogn.gateway.process import string_to_message
from datetime import datetime
from ogn.gateway.process_tools import DummyMerger, Converter, DummySaver
from ogn.gateway.process_tools import DummyMerger, Converter, DbSaver
from ogn.commands.dbutils import session
manager = Manager()
@ -13,7 +13,7 @@ logging_formatstr = '%(asctime)s - %(levelname).4s - %(name)s - %(message)s'
log_levels = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']
# Build the processing pipeline
saver = DummySaver()
saver = DbSaver(session=session)
converter = Converter(callback=saver)
merger = DummyMerger(callback=converter)

4
ogn/gateway/process.py
Wyświetl plik

@ -23,8 +23,8 @@ def _replace_lonlat_with_wkt(message, reference_receiver=None):
distance,bearing = haversine(reference_receiver['latitude'], reference_receiver['longitude'], latitude, longitude)
message['distance'] = distance
message['radial'] = round(bearing)
if message['signal_quality'] is not None and distance >= 1:
message['normalized_signal_quality'] = message['signal_quality'] + 20 * log10(message['distance'] / 10000) # normalized to 10km
if 'signal_quality' in message and message['signal_quality'] is not None and distance >= 1:
message['quality'] = message['signal_quality'] + 20 * log10(message['distance'] / 10000) # normalized to 10km
location = Location(longitude, latitude)
message['location_wkt'] = location.to_wkt()

32
ogn/gateway/process_tools.py
Wyświetl plik

@ -37,13 +37,17 @@ class Merger:
# release messages > max_lines
if self.max_lines is not None:
pass
# merge messages with same timestamp
if message['receiver_name'] in self.message_map:
if message['name'] in self.message_map[message['receiver_name']]:
messages = self.message_map[message['receiver_name']][message['name']]
if message['timestamp'] in messages:
other = messages[message['timestamp']]
receiver_name = message['receiver_name']
name = message['name']
timestamp = message['timestamp']
if receiver_name in self.message_map:
if name in self.message_map[receiver_name]:
timestamps = self.message_map[receiver_name][name]
if timestamp in timestamps:
other = timestamps[timestamp]
params1 = dict( [(k,v) for k,v in message.items() if v is not None])
params2 = dict( [(k,v) for k,v in other.items() if v is not None])
merged = {**params1, **params2}
@ -53,20 +57,20 @@ class Merger:
merged['raw_message'] = '"{}","{}"'.format(message['raw_message'], other['raw_message'])
self.callback.add_message(merged)
del self.message_map[message['receiver_name']][message['name']][message['timestamp']]
del self.message_map[receiver_name][name][timestamp]
else:
self.message_map[message['receiver_name']][message['name']][message['timestamp']] = message
self.message_map[receiver_name][name][timestamp] = message
# release previous messages
for timestamp in list(messages):
if timestamp < message['timestamp']:
self.callback.add_message(messages[timestamp])
del self.message_map[message['receiver_name']][message['name']][timestamp]
for ts in list(timestamps):
if ts < timestamp:
self.callback.add_message(timestamps[ts])
del self.message_map[receiver_name][name][ts]
else:
# add new message
self.message_map[message['receiver_name']].update({message['name']: {message['timestamp']: message}})
self.message_map[receiver_name].update({name: {timestamp: message}})
else:
self.message_map.update({message['receiver_name']: {message['name']: {message['timestamp']: message}}})
self.message_map.update({receiver_name: {name: {timestamp: message}}})
def flush(self):
for receiver,v1 in self.message_map.items():

2
ogn/model/__init__.py
Wyświetl plik

@ -14,5 +14,7 @@ from .takeoff_landing import TakeoffLanding
from .airport import Airport
from .logbook import Logbook
from .receiver_coverage import ReceiverCoverage
from .relation_stats import RelationStats
from .flights2d import Flight2D
from .geo import Location

12
ogn/model/aircraft_beacon.py
Wyświetl plik

@ -44,11 +44,15 @@ class AircraftBeacon(Beacon):
spider_id = None
model = None
status = None
# Naviter stuff
do_not_track = None
reserved = None
# Calculated values
distance = Column(Float(precision=2))
radial = Column(SmallInteger)
normalized_signal_quality = Column(Float(precision=2))
quality = Column(Float(precision=2)) # signal quality normalized to 10km
location_mgrs = Column(String(15))
# Relations
@ -83,7 +87,7 @@ class AircraftBeacon(Beacon):
self.distance,
self.radial,
self.normalized_signal_quality,
self.quality,
self.location_mgrs)
@classmethod
@ -119,7 +123,7 @@ class AircraftBeacon(Beacon):
'distance',
'radial',
'normalized_signal_quality',
'quality',
'location_mgrs']
def get_csv_values(self):
@ -155,7 +159,7 @@ class AircraftBeacon(Beacon):
self.distance,
self.radial,
self.normalized_signal_quality,
self.quality,
self.location_mgrs]

4
ogn/model/device_info.py
Wyświetl plik

@ -1,5 +1,5 @@
from sqlalchemy import Column, Integer, String, Boolean, SmallInteger, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.orm import relationship, backref
from .base import Base
@ -22,7 +22,7 @@ class DeviceInfo(Base):
# Relations
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id], backref='infos')
device = relationship('Device', foreign_keys=[device_id], backref=backref('infos', order_by='DeviceInfo.address_origin.asc()'))
def __repr__(self):
return "<DeviceInfo: %s,%s,%s,%s,%s,%s,%s,%s,%s>" % (

18
ogn/model/device_stats.py
Wyświetl plik

@ -1,5 +1,5 @@
from sqlalchemy import Column, Integer, Date, DateTime, Float, ForeignKey, SmallInteger, Boolean, String
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, Date, DateTime, Float, ForeignKey, SmallInteger, Boolean, String, Index
from sqlalchemy.orm import relationship, backref
from .base import Base
@ -24,8 +24,12 @@ class DeviceStats(Base):
max_altitude = Column(Float(precision=2))
receiver_count = Column(SmallInteger)
aircraft_beacon_count = Column(Integer)
jumps = Column(SmallInteger)
ambiguous = Column(Boolean)
quality = Column(Float(precision=2))
# Relation statistic data
quality_offset = Column(Float(precision=2))
# Ranking data
max_altitude_ranking_worldwide = Column(Integer)
@ -34,14 +38,18 @@ class DeviceStats(Base):
receiver_count_ranking_country = Column(Integer)
aircraft_beacon_count_ranking_worldwide = Column(Integer)
aircraft_beacon_count_ranking_country = Column(Integer)
quality_ranking_worldwide = Column(Integer)
quality_ranking_country = Column(Integer)
# Relations
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id], backref='stats')
device = relationship('Device', foreign_keys=[device_id], backref=backref('stats', order_by='DeviceStats.date.asc()'))
def __repr__(self):
return "<DeviceStats: %s,%s,%s,%s>" % (
self.date,
self.receiver_count,
self.aircraft_beacon_count,
self.max_altitude)
Index('ix_device_stats_date_device_id', DeviceStats.date, DeviceStats.device_id)

27
ogn/model/flights2d.py 100644
Wyświetl plik

@ -0,0 +1,27 @@
from geoalchemy2.types import Geometry
from sqlalchemy import Column, String, Integer, Float, SmallInteger, Date, Index, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base
class Flight2D(Base):
__tablename__ = "flights2d"
id = Column(Integer, primary_key=True)
date = Column(Date)
path_wkt = Column('path', Geometry('MULTILINESTRING', srid=4326))
# Relations
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id], backref='flights2d')
def __repr__(self):
return "<Flight %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,% s>" % (
self.date,
self.path_wkt)
Index('ix_flights2d_date_device_id', Flight2D.date, Flight2D.device_id)
#Index('ix_flights2d_date_path', Flight2D.date, Flight2D.path_wkt) --> CREATE INDEX ix_flights2d_date_path ON flights2d USING GIST("date", path)

4
ogn/model/receiver.py
Wyświetl plik

@ -15,10 +15,10 @@ class Receiver(Base):
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Float(precision=2))
name = Column(String(9))
name = Column(String(9), index=True)
firstseen = Column(DateTime, index=True)
lastseen = Column(DateTime, index=True)
country_code = Column(String(2))
country_code = Column(String(2), index=True)
version = Column(String)
platform = Column(String)

5
ogn/model/receiver_coverage.py
Wyświetl plik

@ -1,5 +1,5 @@
from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index
from sqlalchemy.orm import relationship
from sqlalchemy.orm import relationship, backref
from .base import Base
@ -20,4 +20,5 @@ class ReceiverCoverage(Base):
device_count = Column(SmallInteger)
# Relations
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_coverages')
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref=backref('receiver_coverages', order_by='ReceiverCoverage.date.asc()'))

14
ogn/model/receiver_stats.py
Wyświetl plik

@ -1,5 +1,5 @@
from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String, Index
from sqlalchemy.orm import relationship, backref
from geoalchemy2.types import Geometry
from .base import Base
@ -24,6 +24,10 @@ class ReceiverStats(Base):
aircraft_beacon_count = Column(Integer)
aircraft_count = Column(SmallInteger)
max_distance = Column(Float)
quality = Column(Float(precision=2))
# Relation statistic data
quality_offset = Column(Float(precision=2))
# Ranking data
aircraft_beacon_count_ranking_worldwide = Column(SmallInteger)
@ -32,7 +36,11 @@ class ReceiverStats(Base):
aircraft_count_ranking_country = Column(SmallInteger)
max_distance_ranking_worldwide = Column(SmallInteger)
max_distance_ranking_country = Column(SmallInteger)
quality_ranking_worldwide = Column(Integer)
quality_ranking_country = Column(Integer)
# Relations
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='stats')
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref=backref('stats', order_by='ReceiverStats.date.asc()'))
Index('ix_receiver_stats_date_receiver_id', ReceiverStats.date, ReceiverStats.receiver_id)

31
ogn/model/relation_stats.py 100644
Wyświetl plik

@ -0,0 +1,31 @@
from sqlalchemy import Column, Integer, Date, DateTime, Float, ForeignKey, SmallInteger, Boolean, String, Index
from sqlalchemy.orm import relationship
from .base import Base
class RelationStats(Base):
__tablename__ = "relation_stats"
id = Column(Integer, primary_key=True)
date = Column(Date)
# Statistic data
quality = Column(Float(precision=2))
beacon_count = Column(Integer)
# Relations
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id], backref='relation_stats')
receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='relation_stats')
def __repr__(self):
return "<RelationStats: %s,%s,%s,%s>" % (
self.date,
self.normalized_signal_quality,
self.beacon_count)
Index('ix_relation_stats_date_device_id', RelationStats.date, RelationStats.device_id, RelationStats.receiver_id)
Index('ix_relation_stats_date_receiver_id', RelationStats.date, RelationStats.receiver_id, RelationStats.device_id)

6
ogn/model/takeoff_landing.py
Wyświetl plik

@ -1,5 +1,6 @@
from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey
from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey, Index
from sqlalchemy.orm import relationship
from sqlalchemy.sql import func
from .base import Base
@ -17,3 +18,6 @@ class TakeoffLanding(Base):
# Relations
airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings')
device = relationship('Device', foreign_keys=[device_id], backref='takeoff_landings')
Index('ix_takeoff_landings_date_device_id_airport_id_timestamp', func.date(TakeoffLanding.timestamp), TakeoffLanding.device_id, TakeoffLanding.airport_id, TakeoffLanding.timestamp)
Index('ix_takeoff_landings_date_device_id_timestamp_airport_id', func.date(TakeoffLanding.timestamp), TakeoffLanding.device_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id)

46
ogn/utils.py
Wyświetl plik

@ -7,11 +7,13 @@ from geopy.exc import GeopyError
from geopy.geocoders import Nominatim
from ogn.parser.utils import FEETS_TO_METER
import requests
from urllib.request import Request
from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location
DDB_URL = "http://ddb.glidernet.org/download/?t=1"
FLARMNET_URL = "http://www.flarmnet.org/files/data.fln"
address_prefixes = {'F': 'FLR',
@ -22,12 +24,12 @@ nm2m = 1852
mi2m = 1609.34
def get_ddb(csvfile=None, address_origin=DeviceInfoOrigin.unknown):
if csvfile is None:
def get_ddb(csv_file=None, address_origin=DeviceInfoOrigin.unknown):
if csv_file is None:
r = requests.get(DDB_URL)
rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
else:
r = open(csvfile, 'r')
r = open(csv_file, 'r')
rows = ''.join(i for i in r.readlines() if i[0] != '#')
data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
@ -49,6 +51,25 @@ def get_ddb(csvfile=None, address_origin=DeviceInfoOrigin.unknown):
return device_infos
def get_flarmnet(fln_file=None, address_origin=DeviceInfoOrigin.flarmnet):
if fln_file is None:
r = requests.get(FLARMNET_URL)
rows = [bytes.fromhex(line).decode('latin1') for line in r.text.split('\n') if len(line) == 172]
else:
with open(fln_file, 'r') as file:
rows = [bytes.fromhex(line.strip()).decode('latin1') for line in file.readlines() in len(line) == 172]
device_infos = list()
for row in rows:
device_info = DeviceInfo()
device_info.address = row[0:6].strip()
device_info.aircraft = row[48:69].strip()
device_info.registration = row[69:76].strip()
device_info.competition = row[76:79].strip()
device_infos.append(device_info)
return device_infos
def get_trackable(ddb):
l = []
@ -57,15 +78,28 @@ def get_trackable(ddb):
l.append("{}{}".format(address_prefixes[i.address_type], i.address))
return l
def get_geolocator():
geolocator = Nominatim()
requester = geolocator.urlopen
def requester_hack(req, **kwargs):
req = Request(url=req, headers=geolocator.headers)
return requester(req, **kwargs)
geolocator.urlopen = requester_hack
return geolocator
def get_country_code(latitude, longitude):
geolocator = Nominatim()
geolocator = get_geolocator()
try:
location = geolocator.reverse("{}, {}".format(latitude, longitude))
country_code = location.raw['address']['country_code']
except KeyError:
except KeyError as e:
country_code = None
except GeopyError:
except GeopyError as e:
print(e)
country_code = None
return country_code

14
setup.py
Wyświetl plik

@ -32,16 +32,16 @@ setup(
keywords='gliding ogn',
packages=find_packages(exclude=['tests', 'tests.*']),
install_requires=[
'SQLAlchemy==1.1.15',
'geopy==1.11.0',
'SQLAlchemy==1.2.12',
'geopy==1.17.0',
'manage.py==0.2.10',
'celery[redis]>=3.1,<3.2',
'alembic==0.9.6',
'aerofiles==0.4',
'geoalchemy2==0.4.0',
'alembic==1.0.0',
'aerofiles==0.4.1',
'geoalchemy2==0.5.0',
'shapely>=1.5.17,<1.6',
'ogn-client==0.9.0',
'psycopg2==2.7.3.2',
'ogn-client==0.9.1',
'psycopg2==2.7.5',
'mgrs==1.3.5'
],
extras_require={

9134
tests/data.fln 100644
Wyświetl plik

Plik diff jest za duży Load Diff