Merge pull request #21 from Meisterschueler/fix/travis_build

Fix/travis build
pull/68/head
Meisterschueler 2017-12-13 11:58:09 +01:00 zatwierdzone przez GitHub
commit df8a7346e6
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
23 zmienionych plików z 543 dodań i 295 usunięć

Wyświetl plik

@ -5,18 +5,22 @@ env:
python: python:
- 3.4 - 3.4
- 3.5
- 3.6
services: addons:
- postgresql postgresql: "9.5"
apt:
packages:
- postgresql-9.5-postgis-2.3
before_script: before_script:
- flake8 tests ogn - flake8 tests ogn_test
- psql -c 'CREATE DATABASE ogn_test;' -U postgres - psql -U postgres -c 'CREATE DATABASE ogn_test;'
- psql -c 'CREATE EXTENSION postgis;' -U postgres -d ogn_test - psql -U postgres -c 'CREATE EXTENSION postgis;'
script: script:
- nosetests --with-coverage --cover-package=ogn - nosetests --with-coverage --cover-package=ogn
- pip install . --upgrade - pip install . --upgrade
- python -c 'import ogn' - python -c 'import ogn'

Wyświetl plik

@ -27,8 +27,8 @@ def close_db(signal, sender):
app = Celery('ogn.collect', app = Celery('ogn.collect',
include=["ogn.collect.database", include=["ogn.collect.database",
"ogn.collect.logbook", "ogn.collect.logbook",
"ogn.collect.stats",
"ogn.collect.takeoff_landing", "ogn.collect.takeoff_landing",
"ogn.collect.receiver"
]) ])
app.config_from_envvar("OGN_CONFIG_MODULE") app.config_from_envvar("OGN_CONFIG_MODULE")

Wyświetl plik

@ -1,10 +1,13 @@
from celery.utils.log import get_task_logger from celery.utils.log import get_task_logger
from ogn.model import DeviceInfo, AddressOrigin from sqlalchemy import insert, distinct
from ogn.utils import get_ddb from sqlalchemy.sql import null, and_, or_, func, not_
from sqlalchemy.sql.expression import case
from ogn.collect.celery import app from ogn.model import DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
from ogn.utils import get_ddb, get_country_code
from .celery import app
logger = get_task_logger(__name__) logger = get_task_logger(__name__)
@ -27,7 +30,7 @@ def import_ddb():
"""Import registered devices from the DDB.""" """Import registered devices from the DDB."""
logger.info("Import registered devices fom the DDB...") logger.info("Import registered devices fom the DDB...")
address_origin = AddressOrigin.ogn_ddb address_origin = DeviceInfoOrigin.ogn_ddb
counter = update_device_infos(app.session, address_origin) counter = update_device_infos(app.session, address_origin)
logger.info("Imported {} devices.".format(counter)) logger.info("Imported {} devices.".format(counter))
@ -38,7 +41,212 @@ def import_file(path='tests/custom_ddb.txt'):
"""Import registered devices from a local file.""" """Import registered devices from a local file."""
logger.info("Import registered devices from '{}'...".format(path)) logger.info("Import registered devices from '{}'...".format(path))
address_origin = AddressOrigin.user_defined address_origin = DeviceInfoOrigin.user_defined
counter = update_device_infos(app.session, address_origin, csvfile=path) counter = update_device_infos(app.session, address_origin, csvfile=path)
logger.info("Imported {} devices.".format(counter)) logger.info("Imported {} devices.".format(counter))
@app.task
def update_devices():
"""Add/update entries in devices table and update foreign keys in aircraft beacons."""
# Create missing Device from AircraftBeacon
available_devices = app.session.query(Device.address) \
.subquery()
missing_devices_query = app.session.query(distinct(AircraftBeacon.address)) \
.filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \
.filter(~AircraftBeacon.address.in_(available_devices))
ins = insert(Device).from_select([Device.address], missing_devices_query)
res = app.session.execute(ins)
insert_count = res.rowcount
app.session.commit()
# For each address in the new beacons: get firstseen, lastseen and last values != NULL
last_valid_values = app.session.query(
distinct(AircraftBeacon.address).label('address'),
func.first_value(AircraftBeacon.timestamp)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)]))
.label('firstseen'),
func.last_value(AircraftBeacon.timestamp)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)]))
.label('lastseen'),
func.first_value(AircraftBeacon.aircraft_type)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None), (AircraftBeacon.aircraft_type != null(), AircraftBeacon.aircraft_type)]))
.label('aircraft_type'),
func.first_value(AircraftBeacon.stealth)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None), (AircraftBeacon.stealth != null(), AircraftBeacon.stealth)]))
.label('stealth'),
func.first_value(AircraftBeacon.software_version)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None), (AircraftBeacon.software_version != null(), AircraftBeacon.software_version)]))
.label('software_version'),
func.first_value(AircraftBeacon.hardware_version)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None), (AircraftBeacon.hardware_version != null(), AircraftBeacon.hardware_version)]))
.label('hardware_version'),
func.first_value(AircraftBeacon.real_address)
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)]))
.label('real_address')) \
.filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \
.subquery()
update_values = app.session.query(
Device.address,
case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen),
(Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'),
case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen),
(Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'),
case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type),
(Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'),
case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth),
(Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'),
case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version),
(Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'),
case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version),
(Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'),
case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address),
(Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \
.filter(Device.address == last_valid_values.c.address) \
.subquery()
update_receivers = app.session.query(Device) \
.filter(Device.address == update_values.c.address) \
.update({
Device.firstseen: update_values.c.firstseen,
Device.lastseen: update_values.c.lastseen,
Device.aircraft_type: update_values.c.aircraft_type,
Device.stealth: update_values.c.stealth,
Device.software_version: update_values.c.software_version,
Device.hardware_version: update_values.c.hardware_version,
Device.real_address: update_values.c.real_address},
synchronize_session='fetch')
# Update relations to aircraft beacons
upd = app.session.query(AircraftBeacon) \
.filter(AircraftBeacon.device_id == null()) \
.filter(AircraftBeacon.address == Device.address) \
.update({
AircraftBeacon.device_id: Device.id},
synchronize_session='fetch')
app.session.commit()
logger.info("Devices: {} inserted, {} updated".format(insert_count, update_receivers))
logger.info("Updated {} AircraftBeacons".format(upd))
return "{} Devices inserted, {} Devices updated, {} AircraftBeacons updated" \
.format(insert_count, update_receivers, upd)
@app.task
def update_receivers():
"""Add/update_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons."""
# Create missing Receiver from ReceiverBeacon
available_receivers = app.session.query(Receiver.name) \
.subquery()
missing_receiver_query = app.session.query(distinct(ReceiverBeacon.name)) \
.filter(ReceiverBeacon.receiver_id == null()) \
.filter(~ReceiverBeacon.name.in_(available_receivers))
ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query)
res = app.session.execute(ins)
insert_count = res.rowcount
# For each name in the new beacons: get firstseen, lastseen and last values != NULL
last_valid_values = app.session.query(
distinct(ReceiverBeacon.name).label('name'),
func.first_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)]))
.label('firstseen'),
func.last_value(ReceiverBeacon.timestamp)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)]))
.label('lastseen'),
func.first_value(ReceiverBeacon.location_wkt)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None), (ReceiverBeacon.location_wkt != null(), ReceiverBeacon.location_wkt)]))
.label('location_wkt'),
func.first_value(ReceiverBeacon.altitude)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None), (ReceiverBeacon.altitude != null(), ReceiverBeacon.altitude)]))
.label('altitude'),
func.first_value(ReceiverBeacon.version)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None), (ReceiverBeacon.version != null(), ReceiverBeacon.version)]))
.label('version'),
func.first_value(ReceiverBeacon.platform)
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None), (ReceiverBeacon.platform != null(), ReceiverBeacon.platform)]))
.label('platform')) \
.filter(ReceiverBeacon.receiver_id == null()) \
.subquery()
update_values = app.session.query(
Receiver.name,
case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen),
(Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'),
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen),
(Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'),
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)),
(Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'),
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude),
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'),
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version),
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'),
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform),
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'),
case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed
(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \
.filter(Receiver.name == last_valid_values.c.name) \
.subquery()
update_receivers = app.session.query(Receiver) \
.filter(Receiver.name == update_values.c.name) \
.update({
Receiver.firstseen: update_values.c.firstseen,
Receiver.lastseen: update_values.c.lastseen,
Receiver.location_wkt: update_values.c.location_wkt,
Receiver.altitude: update_values.c.altitude,
Receiver.version: update_values.c.version,
Receiver.platform: update_values.c.platform,
Receiver.country_code: update_values.c.country_code},
synchronize_session='fetch')
# Update relations to aircraft beacons
update_aircraft_beacons = app.session.query(AircraftBeacon) \
.filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \
.update({AircraftBeacon.receiver_id: Receiver.id,
AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)},
synchronize_session='fetch')
# Update relations to receiver beacons
update_receiver_beacons = app.session.query(ReceiverBeacon) \
.filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \
.update({ReceiverBeacon.receiver_id: Receiver.id},
synchronize_session='fetch')
app.session.commit()
logger.info("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers))
logger.info("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons))
return "{} Receivers inserted, {} Receivers updated, {} AircraftBeacons updated, {} ReceiverBeacons updated" \
.format(insert_count, update_receivers, update_aircraft_beacons, update_receiver_beacons)
@app.task
def update_country_code():
# update country code in receivers table if None
unknown_country_query = app.session.query(Receiver) \
.filter(Receiver.country_code == null()) \
.filter(Receiver.location_wkt != null()) \
.order_by(Receiver.name)
counter = 0
for receiver in unknown_country_query.all():
location = receiver.location
country_code = get_country_code(location.latitude, location.longitude)
if country_code is not None:
receiver.country_code = country_code
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
counter += 1
app.session.commit()
return "Updated country_code for {} Receivers".format(counter)

Wyświetl plik

@ -1,25 +1,22 @@
from celery.utils.log import get_task_logger from celery.utils.log import get_task_logger
from sqlalchemy import and_, or_, insert, update, between, exists from sqlalchemy import and_, or_, insert, update, exists
from sqlalchemy.sql import func, null from sqlalchemy.sql import func, null
from sqlalchemy.sql.expression import true, false from sqlalchemy.sql.expression import true, false
from ogn.collect.celery import app from ogn.collect.celery import app
from ogn.model import TakeoffLanding, Logbook from ogn.model import TakeoffLanding, Logbook, AircraftBeacon
logger = get_task_logger(__name__) logger = get_task_logger(__name__)
@app.task @app.task
def compute_logbook_entries(session=None): def update_logbook(session=None):
logger.info("Compute logbook.") logger.info("Compute logbook.")
if session is None: if session is None:
session = app.session session = app.session
or_args = [between(TakeoffLanding.timestamp, '2016-06-28 00:00:00', '2016-06-28 23:59:59')]
or_args = []
# 'wo' is the window order for the sql window function # 'wo' is the window order for the sql window function
wo = and_(func.date(TakeoffLanding.timestamp), wo = and_(func.date(TakeoffLanding.timestamp),
TakeoffLanding.device_id, TakeoffLanding.device_id,
@ -43,7 +40,6 @@ def compute_logbook_entries(session=None):
TakeoffLanding.airport_id, TakeoffLanding.airport_id,
func.lag(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_prev'), func.lag(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_prev'),
func.lead(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_next')) \ func.lead(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_next')) \
.filter(*or_args) \
.subquery() .subquery()
# find complete flights (with takeoff and landing on the same day) # find complete flights (with takeoff and landing on the same day)
@ -156,4 +152,36 @@ def compute_logbook_entries(session=None):
session.commit() session.commit()
logger.debug("New logbook entries: {}".format(insert_counter)) logger.debug("New logbook entries: {}".format(insert_counter))
return "{}/{}".format(update_counter, insert_counter) return "Logbook entries: {} inserted, {} updated".format(update_counter, insert_counter)
@app.task
def update_max_altitude(session=None):
logger.info("Update logbook max altitude.")
if session is None:
session = app.session
logbook_entries = session.query(Logbook.id) \
.filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null())) \
.limit(1000) \
.subquery()
max_altitudes = session.query(Logbook.id, func.max(AircraftBeacon.altitude).label('max_altitude')) \
.filter(Logbook.id == logbook_entries.c.id) \
.filter(and_(AircraftBeacon.device_id == Logbook.device_id,
AircraftBeacon.timestamp >= Logbook.takeoff_timestamp,
AircraftBeacon.timestamp <= Logbook.landing_timestamp)) \
.group_by(Logbook.id) \
.subquery()
update_logbook = app.session.query(Logbook) \
.filter(Logbook.id == max_altitudes.c.id) \
.update({
Logbook.max_altitude: max_altitudes.c.max_altitude},
synchronize_session='fetch')
session.commit()
logger.info("Logbook: {} entries updated.".format(update_logbook))
return "Logbook: {} entries updated.".format(update_logbook)

Wyświetl plik

@ -1,111 +0,0 @@
from sqlalchemy.sql import func, null
from sqlalchemy.sql.functions import coalesce
from sqlalchemy import and_, not_, or_
from celery.utils.log import get_task_logger
from ogn.model import Receiver, ReceiverBeacon
from ogn.utils import get_country_code
from ogn.collect.celery import app
logger = get_task_logger(__name__)
@app.task
def update_receivers():
"""Update the receiver table."""
# get the timestamp of last update
last_update_query = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry'))
last_update = last_update_query.one().last_entry
# get last receiver beacons since last update
last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name,
func.max(ReceiverBeacon.timestamp).label('lastseen')) \
.filter(ReceiverBeacon.timestamp >= last_update) \
.group_by(ReceiverBeacon.name) \
.subquery()
# update receivers
receivers_to_update = app.session.query(ReceiverBeacon.name,
ReceiverBeacon.location_wkt,
ReceiverBeacon.altitude,
last_receiver_beacon_sq.columns.lastseen,
ReceiverBeacon.version,
ReceiverBeacon.platform) \
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
.subquery()
# ... set country code to None if lat or lon changed
changed_count = app.session.query(Receiver) \
.filter(Receiver.name == receivers_to_update.columns.name) \
.filter(or_(not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)),
and_(Receiver.location_wkt == null(),
receivers_to_update.columns.location != null()))) \
.update({"location_wkt": receivers_to_update.columns.location,
"country_code": null()},
synchronize_session=False)
# ... and update altitude, lastseen, version and platform
update_count = app.session.query(Receiver) \
.filter(Receiver.name == receivers_to_update.columns.name) \
.update({"altitude": receivers_to_update.columns.altitude,
"lastseen": receivers_to_update.columns.lastseen,
"version": receivers_to_update.columns.version,
"platform": receivers_to_update.columns.platform})
# add new receivers
empty_sq = app.session.query(ReceiverBeacon.name,
ReceiverBeacon.location_wkt,
ReceiverBeacon.altitude,
last_receiver_beacon_sq.columns.lastseen,
ReceiverBeacon.version, ReceiverBeacon.platform) \
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
.outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \
.filter(Receiver.name == null()) \
.order_by(ReceiverBeacon.name)
for receiver_beacon in empty_sq.all():
receiver = Receiver()
receiver.name = receiver_beacon.name
receiver.location_wkt = receiver_beacon.location_wkt
receiver.altitude = receiver_beacon.altitude
receiver.firstseen = None
receiver.lastseen = receiver_beacon.lastseen
receiver.version = receiver_beacon.version
receiver.platform = receiver_beacon.platform
app.session.add(receiver)
logger.info("{} added".format(receiver.name))
# update firstseen if None
firstseen_null_query = app.session.query(Receiver.name,
func.min(ReceiverBeacon.timestamp).label('firstseen')) \
.filter(Receiver.firstseen == null()) \
.join(ReceiverBeacon, Receiver.name == ReceiverBeacon.name) \
.group_by(Receiver.name) \
.subquery()
added_count = app.session.query(Receiver) \
.filter(Receiver.name == firstseen_null_query.columns.name) \
.update({'firstseen': firstseen_null_query.columns.firstseen})
# update country code if None
unknown_country_query = app.session.query(Receiver) \
.filter(Receiver.country_code == null()) \
.filter(Receiver.location_wkt != null()) \
.order_by(Receiver.name)
for receiver in unknown_country_query.all():
location = receiver.location
country_code = get_country_code(location.latitude, location.longitude)
if country_code is not None:
receiver.country_code = country_code
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
logger.info("Added: {}, location changed: {}".format(added_count, changed_count))
app.session.commit()
return update_count

Wyświetl plik

@ -0,0 +1,85 @@
from celery.utils.log import get_task_logger
from sqlalchemy import insert, distinct
from sqlalchemy.sql import null, and_, func
from sqlalchemy.sql.expression import literal_column
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
from .celery import app
logger = get_task_logger(__name__)
@app.task
def update_device_stats(date=None):
"""Add/update entries in device stats table."""
if not date:
logger.warn("A date is needed for calculating stats. Exiting")
return None
# First kill the stats for the selected date
deleted_counter = app.session.query(DeviceStats) \
.filter(DeviceStats.date == date) \
.delete()
# Calculate stats for the selected date
device_stats = app.session.query(
AircraftBeacon.device_id,
func.date(AircraftBeacon.timestamp).label('date'),
func.count(distinct(AircraftBeacon.receiver_id)).label('receiver_count'),
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
func.max(AircraftBeacon.altitude).label('max_altitude')) \
.filter(and_(AircraftBeacon.device_id != null(), AircraftBeacon.receiver_id != null())) \
.filter(func.date(AircraftBeacon.timestamp) == date) \
.group_by(AircraftBeacon.device_id, func.date(AircraftBeacon.timestamp)) \
.subquery()
# And insert them
ins = insert(DeviceStats).from_select(
[DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.aircraft_beacon_count, DeviceStats.max_altitude],
device_stats)
res = app.session.execute(ins)
insert_counter = res.rowcount
app.session.commit()
logger.debug("DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
return "DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
@app.task
def update_receiver_stats(date=None):
"""Add/update entries in receiver stats table."""
if not date:
logger.warn("A date is needed for calculating stats. Exiting")
return None
# First kill the stats for the selected date
deleted_counter = app.session.query(ReceiverStats) \
.filter(ReceiverStats.date == date) \
.delete()
# Calculate stats for the selected date
receiver_stats = app.session.query(
AircraftBeacon.receiver_id,
literal_column("'{}'".format(date)).label('date'),
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'),
func.max(AircraftBeacon.distance).label('max_distance')) \
.filter(AircraftBeacon.receiver_id != null()) \
.filter(func.date(AircraftBeacon.timestamp) == date) \
.group_by(AircraftBeacon.receiver_id) \
.subquery()
# And insert them
ins = insert(ReceiverStats).from_select(
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance],
receiver_stats)
res = app.session.execute(ins)
insert_counter = res.rowcount
app.session.commit()
logger.debug("ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
return "ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)

Wyświetl plik

@ -13,7 +13,7 @@ logger = get_task_logger(__name__)
@app.task @app.task
def compute_takeoff_and_landing(session=None): def update_takeoff_landing(session=None):
logger.info("Compute takeoffs and landings.") logger.info("Compute takeoffs and landings.")
if session is None: if session is None:
@ -29,10 +29,10 @@ def compute_takeoff_and_landing(session=None):
takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit
landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit
duration = 100 # the points must not exceed this duration duration = 100 # the points must not exceed this duration
radius = 0.05 # the points must not exceed this radius (degree!) around the 2nd point radius = 5000 # the points must not exceed this radius around the 2nd point
# takeoff / landing has to be near an airport # takeoff / landing has to be near an airport
airport_radius = 0.025 # takeoff / landing must not exceed this radius (degree!) around the airport airport_radius = 2500 # takeoff / landing must not exceed this radius around the airport
airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport
# 'wo' is the window order for the sql window function # 'wo' is the window order for the sql window function
@ -41,8 +41,19 @@ def compute_takeoff_and_landing(session=None):
AircraftBeacon.receiver_id) AircraftBeacon.receiver_id)
# make a query with current, previous and next position # make a query with current, previous and next position
beacon_selection = session.query(AircraftBeacon.id) \
.filter(AircraftBeacon.status == null()) \
.order_by(AircraftBeacon.timestamp) \
.limit(1000000) \
.subquery()
sq = session.query( sq = session.query(
AircraftBeacon.id, AircraftBeacon.id,
func.lag(AircraftBeacon.id).over(order_by=wo).label('id_prev'),
func.lead(AircraftBeacon.id).over(order_by=wo).label('id_next'),
AircraftBeacon.device_id,
func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'),
func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next'),
AircraftBeacon.timestamp, AircraftBeacon.timestamp,
func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'), func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'),
func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'), func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'),
@ -57,16 +68,14 @@ def compute_takeoff_and_landing(session=None):
func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'), func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'),
AircraftBeacon.altitude, AircraftBeacon.altitude,
func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'), func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'),
func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next'), func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next')) \
AircraftBeacon.device_id, .filter(AircraftBeacon.id == beacon_selection.c.id) \
func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'),
func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next')) \
.filter(AircraftBeacon.status == null()) \
.subquery() .subquery()
# consider only positions with the same device id
sq2 = session.query(sq) \ sq2 = session.query(sq) \
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
.subquery() .subquery()
# find possible takeoffs and landings # find possible takeoffs and landings
sq3 = session.query( sq3 = session.query(
@ -82,8 +91,8 @@ def compute_takeoff_and_landing(session=None):
(sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'), (sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'),
sq2.c.device_id) \ sq2.c.device_id) \
.filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \ .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \
.filter(and_(func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_prev, radius), .filter(and_(func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_prev) < radius,
func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_next, radius))) \ func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_next) < radius)) \
.filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff .filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff
sq2.c.ground_speed > takeoff_speed, sq2.c.ground_speed > takeoff_speed,
sq2.c.ground_speed_next > takeoff_speed), sq2.c.ground_speed_next > takeoff_speed),
@ -99,7 +108,7 @@ def compute_takeoff_and_landing(session=None):
sq3.c.is_takeoff, sq3.c.is_takeoff,
sq3.c.device_id, sq3.c.device_id,
Airport.id.label('airport_id')) \ Airport.id.label('airport_id')) \
.filter(and_(func.ST_DFullyWithin(sq3.c.location, Airport.location_wkt, airport_radius), .filter(and_(func.ST_Distance_Sphere(sq3.c.location, Airport.location_wkt) < airport_radius,
between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \
.filter(between(Airport.style, 2, 5)) \ .filter(between(Airport.style, 2, 5)) \
.subquery() .subquery()
@ -120,7 +129,14 @@ def compute_takeoff_and_landing(session=None):
takeoff_landing_query) takeoff_landing_query)
result = session.execute(ins) result = session.execute(ins)
counter = result.rowcount counter = result.rowcount
# mark the computated AircraftBeacons as 'used'
update_aircraft_beacons = session.query(AircraftBeacon) \
.filter(AircraftBeacon.id == sq2.c.id) \
.update({AircraftBeacon.status: 1},
synchronize_session='fetch')
session.commit() session.commit()
logger.debug("New takeoffs and landings: {}".format(counter)) logger.debug("Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(counter, update_aircraft_beacons))
return counter return counter

Wyświetl plik

@ -1,4 +1,5 @@
import os import os
import re
from manager import Manager from manager import Manager
from ogn.commands.dbutils import session from ogn.commands.dbutils import session
@ -9,18 +10,19 @@ from ogn.utils import open_file
manager = Manager() manager = Manager()
PATTERN = '^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$'
@manager.command @manager.command
def convert_logfile(path, logfile='main.log', loglevel='INFO'): def convert_logfile(path, logfile='main.log', loglevel='INFO'):
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD.""" """Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
if os.path.isfile(path): if os.path.isfile(path):
print("Reading file: {}".format(path)) head, tail = os.path.split(path)
convert(path) convert(tail, path=head)
print("Finished") print("Finished")
elif os.path.isdir(path): elif os.path.isdir(path):
for filename in os.listdir(path): for filename in os.listdir(path):
print("Reading file: {}".format(filename))
convert(filename, path=path) convert(filename, path=path)
print("Finished") print("Finished")
else: else:
@ -28,15 +30,25 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'):
def convert(sourcefile, path=''): def convert(sourcefile, path=''):
import re
import csv import csv
import gzip import gzip
import datetime import datetime
match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile) match = re.search(PATTERN, sourcefile)
if match: if match:
reference_date_string = match.group(1) reference_date_string = match.group(1)
reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d") reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d")
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
print("Reading file: {}".format(sourcefile))
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
else:
print("Output files for file {} already exists. Skipping".format(sourcefile))
return
else: else:
print("filename '{}' does not match pattern. Skipping".format(sourcefile)) print("filename '{}' does not match pattern. Skipping".format(sourcefile))
return return
@ -49,16 +61,6 @@ def convert(sourcefile, path=''):
total += 1 total += 1
fin.seek(0) fin.seek(0)
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
else:
print("Output files already exists. Skipping")
return
aircraft_beacons = list() aircraft_beacons = list()
receiver_beacons = list() receiver_beacons = list()
@ -182,7 +184,7 @@ def import_logfile(path):
else: else:
print("For {} beacons already exist. Skipping".format(reference_date_string)) print("For {} beacons already exist. Skipping".format(reference_date_string))
else: else:
print("Unknown file type: {}".format()) print("Unknown file type: {}".format(tail))
def check_no_beacons(tablename, reference_date_string): def check_no_beacons(tablename, reference_date_string):

Wyświetl plik

@ -1,10 +1,10 @@
from manager import Manager from manager import Manager
from ogn.collect.database import update_device_infos from ogn.collect.database import update_device_infos
from ogn.commands.dbutils import engine, session from ogn.commands.dbutils import engine, session
from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon
from ogn.utils import get_airports from ogn.utils import get_airports
from sqlalchemy import insert, distinct from sqlalchemy import distinct
from sqlalchemy.sql import null from sqlalchemy.sql import null, func
manager = Manager() manager = Manager()
@ -53,7 +53,7 @@ def import_ddb():
"""Import registered devices from the DDB.""" """Import registered devices from the DDB."""
print("Import registered devices fom the DDB...") print("Import registered devices fom the DDB...")
address_origin = AddressOrigin.ogn_ddb address_origin = DeviceInfoOrigin.ogn_ddb
counter = update_device_infos(session, counter = update_device_infos(session,
address_origin) address_origin)
print("Imported %i devices." % counter) print("Imported %i devices." % counter)
@ -65,7 +65,7 @@ def import_file(path='tests/custom_ddb.txt'):
# (flushes previously manually imported entries) # (flushes previously manually imported entries)
print("Import registered devices from '{}'...".format(path)) print("Import registered devices from '{}'...".format(path))
address_origin = AddressOrigin.user_defined address_origin = DeviceInfoOrigin.user_defined
counter = update_device_infos(session, counter = update_device_infos(session,
address_origin, address_origin,
csvfile=path) csvfile=path)
@ -81,53 +81,3 @@ def import_airports(path='tests/SeeYou.cup'):
session.bulk_save_objects(airports) session.bulk_save_objects(airports)
session.commit() session.commit()
print("Imported {} airports.".format(len(airports))) print("Imported {} airports.".format(len(airports)))
@manager.command
def update_relations():
"""Update AircraftBeacon and ReceiverBeacon relations"""
# Create missing Receiver from ReceiverBeacon
available_receivers = session.query(Receiver.name) \
.subquery()
missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \
.filter(ReceiverBeacon.receiver_id == null()) \
.filter(~ReceiverBeacon.name.in_(available_receivers))
ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query)
session.execute(ins)
# Create missing Device from AircraftBeacon
available_addresses = session.query(Device.address) \
.subquery()
missing_addresses_query = session.query(distinct(AircraftBeacon.address)) \
.filter(AircraftBeacon.device_id == null()) \
.filter(~AircraftBeacon.address.in_(available_addresses))
ins2 = insert(Device).from_select([Device.address], missing_addresses_query)
session.execute(ins2)
session.commit()
print("Inserted {} Receivers and {} Devices".format(ins, ins2))
return
# Update AircraftBeacons
upd = session.query(AircraftBeacon) \
.filter(AircraftBeacon.device_id == null()) \
.filter(AircraftBeacon.receiver_id == null()) \
.filter(AircraftBeacon.address == Device.address) \
.filter(AircraftBeacon.receiver_name == Receiver.name) \
.update({AircraftBeacon.device_id: Device.id,
AircraftBeacon.receiver_id: Receiver.id},
synchronize_session='fetch')
upd2 = session.query(ReceiverBeacon) \
.filter(ReceiverBeacon.receiver_id == null()) \
.filter(ReceiverBeacon.receiver_name == Receiver.name) \
.update({Receiver.name: ReceiverBeacon.receiver_name},
synchronize_session='fetch')
session.commit()
print("Updated {} AircraftBeacons and {} ReceiverBeacons".
format(upd, upd2))

Wyświetl plik

@ -3,8 +3,8 @@
from datetime import timedelta, datetime from datetime import timedelta, datetime
from manager import Manager from manager import Manager
from ogn.collect.logbook import compute_logbook_entries from ogn.collect.logbook import update_logbook
from ogn.collect.takeoff_landing import compute_takeoff_and_landing from ogn.collect.takeoff_landing import update_takeoff_landing
from ogn.commands.dbutils import session from ogn.commands.dbutils import session
from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook
from sqlalchemy import and_, or_ from sqlalchemy import and_, or_
@ -19,7 +19,7 @@ manager = Manager()
def compute_takeoff_landing(): def compute_takeoff_landing():
"""Compute takeoffs and landings.""" """Compute takeoffs and landings."""
print("Compute takeoffs and landings...") print("Compute takeoffs and landings...")
result = compute_takeoff_and_landing.delay() result = update_takeoff_landing.delay()
counter = result.get() counter = result.get()
print("New takeoffs/landings: {}".format(counter)) print("New takeoffs/landings: {}".format(counter))
@ -28,7 +28,7 @@ def compute_takeoff_landing():
def compute_logbook(): def compute_logbook():
"""Compute logbook.""" """Compute logbook."""
print("Compute logbook...") print("Compute logbook...")
result = compute_logbook_entries.delay() result = update_logbook.delay()
counter = result.get() counter = result.get()
print("New logbook entries: {}".format(counter)) print("New logbook entries: {}".format(counter))
@ -63,7 +63,8 @@ def show(airport_name, utc_delta_hours=0, date=None):
# get all logbook entries and add device and airport infos # get all logbook entries and add device and airport infos
takeoff_airport = aliased(Airport, name='takeoff_airport') takeoff_airport = aliased(Airport, name='takeoff_airport')
landing_airport = aliased(Airport, name='landing_airport') landing_airport = aliased(Airport, name='landing_airport')
logbook_query = session.query(Logbook, logbook_query = session.query(func.row_number().over(order_by=Logbook.reftime).label('row_number'),
Logbook,
Device, Device,
sq3.c.registration, sq3.c.registration,
sq3.c.aircraft) \ sq3.c.aircraft) \
@ -105,8 +106,9 @@ def show(airport_name, utc_delta_hours=0, date=None):
def none_altitude_replacer(altitude_object, airport_object): def none_altitude_replacer(altitude_object, airport_object):
return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude) return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude)
for [logbook, device, registration, aircraft] in logbook_query.all(): for [row_number, logbook, device, registration, aircraft] in logbook_query.all():
print('%10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % ( print('%3d. %10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % (
row_number,
logbook.reftime.date(), logbook.reftime.date(),
none_datetime_replacer(logbook.takeoff_timestamp), none_datetime_replacer(logbook.takeoff_timestamp),
none_track_replacer(logbook.takeoff_track), none_track_replacer(logbook.takeoff_track),

Wyświetl plik

@ -1,6 +1,6 @@
from manager import Manager from manager import Manager
from ogn.commands.dbutils import session from ogn.commands.dbutils import session
from ogn.model import AddressOrigin from ogn.model import DeviceInfoOrigin
from ogn.model.device_info import DeviceInfo from ogn.model.device_info import DeviceInfo
from sqlalchemy import func, and_, true, false from sqlalchemy import func, and_, true, false
@ -39,7 +39,7 @@ def get_devices_stats(session):
stats = {} stats = {}
for [address_origin, device_count, default_count, nt_count, ni_count, ntni_count] in query.all(): for [address_origin, device_count, default_count, nt_count, ni_count, ntni_count] in query.all():
origin = AddressOrigin(address_origin).name() origin = DeviceInfoOrigin(address_origin).name()
stats[origin] = {'device_count': device_count, stats[origin] = {'device_count': device_count,
'default_count': default_count, 'default_count': default_count,
'nt_count': nt_count, 'nt_count': nt_count,

Wyświetl plik

@ -31,6 +31,9 @@ def message_to_beacon(raw_message, reference_date):
beacon = ReceiverBeacon(**message) beacon = ReceiverBeacon(**message)
else: else:
print("Whoops: what is this: {}".format(message)) print("Whoops: what is this: {}".format(message))
except NotImplementedError as e:
logger.error('Received message: {}'.format(raw_message))
logger.error(e)
except ParseError as e: except ParseError as e:
logger.error('Received message: {}'.format(raw_message)) logger.error('Received message: {}'.format(raw_message))
logger.error('Drop packet, {}'.format(e.message)) logger.error('Drop packet, {}'.format(e.message))

Wyświetl plik

@ -1,13 +1,15 @@
# flake8: noqa # flake8: noqa
from .address_origin import AddressOrigin
from .aircraft_type import AircraftType from .aircraft_type import AircraftType
from .base import Base from .base import Base
from .beacon import Beacon from .beacon import Beacon
from .device import Device from .device import Device
from .device_info import DeviceInfo from .device_info import DeviceInfo
from .device_info_origin import DeviceInfoOrigin
from .device_stats import DeviceStats
from .aircraft_beacon import AircraftBeacon from .aircraft_beacon import AircraftBeacon
from .receiver_beacon import ReceiverBeacon from .receiver_beacon import ReceiverBeacon
from .receiver import Receiver from .receiver import Receiver
from .receiver_stats import ReceiverStats
from .takeoff_landing import TakeoffLanding from .takeoff_landing import TakeoffLanding
from .airport import Airport from .airport import Airport
from .logbook import Logbook from .logbook import Logbook

Wyświetl plik

@ -38,6 +38,9 @@ class AircraftBeacon(Beacon):
status = Column(SmallInteger, index=True) status = Column(SmallInteger, index=True)
# Calculated values
distance = Column(Float)
# Relations # Relations
receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True) receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)
receiver = relationship('Receiver', foreign_keys=[receiver_id]) receiver = relationship('Receiver', foreign_keys=[receiver_id])

Wyświetl plik

@ -1,4 +1,4 @@
from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger, DateTime
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from .base import Base from .base import Base
@ -9,11 +9,15 @@ class Device(Base):
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
address = Column(String(6), index=True) address = Column(String(6), index=True)
firstseen = Column(DateTime, index=True)
lastseen = Column(DateTime, index=True)
aircraft_type = Column(SmallInteger, index=True) aircraft_type = Column(SmallInteger, index=True)
stealth = Column(Boolean) stealth = Column(Boolean)
software_version = Column(Float) software_version = Column(Float)
hardware_version = Column(SmallInteger) hardware_version = Column(SmallInteger)
real_address = Column(String(6)) real_address = Column(String(6))
firstseen = Column(DateTime, index=True)
lastseen = Column(DateTime, index=True)
# Relations # Relations
aircraft_beacons = relationship('AircraftBeacon') aircraft_beacons = relationship('AircraftBeacon')

Wyświetl plik

@ -1,4 +1,4 @@
class AddressOrigin: class DeviceInfoOrigin:
unknown = 0 unknown = 0
ogn_ddb = 1 ogn_ddb = 1
flarmnet = 2 flarmnet = 2

Wyświetl plik

@ -0,0 +1,19 @@
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base
class DeviceStats(Base):
__tablename__ = "device_stats"
id = Column(Integer, primary_key=True)
date = Column(Date)
receiver_count = Column(Integer)
aircraft_beacon_count = Column(Integer)
max_altitude = Column(Float)
# Relations
device_id = Column(Integer, ForeignKey('device.id', ondelete='SET NULL'), index=True)
device = relationship('Device', foreign_keys=[device_id])

Wyświetl plik

@ -0,0 +1,20 @@
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base
class ReceiverStats(Base):
__tablename__ = "receiver_stats"
id = Column(Integer, primary_key=True)
date = Column(Date)
aircraft_beacon_count = Column(Integer)
receiver_beacon_count = Column(Integer)
aircraft_count = Column(Integer)
max_distance = Column(Float)
# Relations
receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)
receiver = relationship('Receiver', foreign_keys=[receiver_id])

Wyświetl plik

@ -8,7 +8,7 @@ from geopy.geocoders import Nominatim
from ogn.parser.utils import feet2m from ogn.parser.utils import feet2m
import requests import requests
from .model import AddressOrigin, DeviceInfo, Airport, Location from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location
DDB_URL = "http://ddb.glidernet.org/download/?t=1" DDB_URL = "http://ddb.glidernet.org/download/?t=1"
@ -22,7 +22,7 @@ nm2m = 1852
mi2m = 1609.34 mi2m = 1609.34
def get_ddb(csvfile=None, address_origin=AddressOrigin.unknown): def get_ddb(csvfile=None, address_origin=DeviceInfoOrigin.unknown):
if csvfile is None: if csvfile is None:
r = requests.get(DDB_URL) r = requests.get(DDB_URL)
rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#') rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')

Wyświetl plik

@ -32,22 +32,22 @@ setup(
keywords='gliding ogn', keywords='gliding ogn',
packages=find_packages(exclude=['tests', 'tests.*']), packages=find_packages(exclude=['tests', 'tests.*']),
install_requires=[ install_requires=[
'SQLAlchemy==1.1.10', 'SQLAlchemy==1.1.15',
'geopy==1.11.0', 'geopy==1.11.0',
'manage.py==0.2.10', 'manage.py==0.2.10',
'celery[redis]>=3.1,<3.2', 'celery[redis]>=3.1,<3.2',
'alembic==0.9.2', 'alembic==0.9.6',
'aerofiles==0.3', 'aerofiles==0.4',
'geoalchemy2==0.4.0', 'geoalchemy2==0.4.0',
'shapely==1.5.17.post1', 'shapely>=1.5.17,<1.6',
'ogn-client==0.8.0', 'ogn-client==0.8.0',
'psycopg2==2.7.1' 'psycopg2==2.7.3.2'
], ],
extras_require={ extras_require={
'dev': [ 'dev': [
'nose==1.3.7', 'nose==1.3.7',
'coveralls==1.1', 'coveralls==1.2',
'flake8==3.3.0' 'flake8==3.5.0'
] ]
}, },
zip_safe=False zip_safe=False

Wyświetl plik

@ -1,7 +1,10 @@
import unittest import unittest
import os import os
from ogn.collect.logbook import compute_logbook_entries from sqlalchemy.sql import null, and_
from ogn.model import Logbook, Airport
from ogn.collect.logbook import update_logbook
class TestDB(unittest.TestCase): class TestDB(unittest.TestCase):
@ -35,17 +38,43 @@ class TestDB(unittest.TestCase):
session.commit() session.commit()
pass pass
def count_logbook_entries(self):
session = self.session
query = session.query(Logbook)
return len(query.all())
def assert_entries(self, koen_to=0, koen_ldg=0, koen_complete=0, ohl_to=0, ohl_ldg=0, ohl_complete=0):
session = self.session
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.landing_airport_id == null()).all())
self.assertEqual(entries, koen_to)
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == null()).all())
self.assertEqual(entries, koen_ldg)
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all())
self.assertEqual(entries, koen_complete)
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.landing_airport_id == null()).all())
self.assertEqual(entries, ohl_to)
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == null()).all())
self.assertEqual(entries, ohl_ldg)
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all())
self.assertEqual(entries, ohl_complete)
def test_single_takeoff(self): def test_single_takeoff(self):
session = self.session session = self.session
session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_to=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_to=1)
def test_single_landing(self): def test_single_landing(self):
session = self.session session = self.session
@ -53,11 +82,11 @@ class TestDB(unittest.TestCase):
session.execute(self.LANDING_KOENIGSDF_DD0815) session.execute(self.LANDING_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_ldg=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_ldg=1)
def test_different_takeoffs(self): def test_different_takeoffs(self):
session = self.session session = self.session
@ -66,11 +95,11 @@ class TestDB(unittest.TestCase):
session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.execute(self.TAKEOFF_OHLSTADT_DD4711)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/2') self.assert_entries(koen_to=1, ohl_to=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_to=1, ohl_to=1)
def test_takeoff_and_landing(self): def test_takeoff_and_landing(self):
session = self.session session = self.session
@ -79,11 +108,11 @@ class TestDB(unittest.TestCase):
session.execute(self.LANDING_KOENIGSDF_DD0815) session.execute(self.LANDING_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_complete=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_complete=1)
def test_takeoff_and_landing_on_different_days(self): def test_takeoff_and_landing_on_different_days(self):
session = self.session session = self.session
@ -92,11 +121,11 @@ class TestDB(unittest.TestCase):
session.execute(self.LANDING_KOENIGSDF_DD0815_LATER) session.execute(self.LANDING_KOENIGSDF_DD0815_LATER)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/2') self.assert_entries(koen_to=1, koen_ldg=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_to=1, koen_ldg=1)
def test_update(self): def test_update(self):
session = self.session session = self.session
@ -104,23 +133,23 @@ class TestDB(unittest.TestCase):
session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_to=1)
session.execute(self.LANDING_KOENIGSDF_DD0815) session.execute(self.LANDING_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '1/0') self.assert_entries(koen_complete=1)
session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.execute(self.TAKEOFF_OHLSTADT_DD4711)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_complete=1, ohl_to=1)
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/0') self.assert_entries(koen_complete=1, ohl_to=1)
def test_update_wrong_order(self): def test_update_wrong_order(self):
session = self.session session = self.session
@ -128,14 +157,14 @@ class TestDB(unittest.TestCase):
session.execute(self.LANDING_KOENIGSDF_DD0815) session.execute(self.LANDING_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '0/1') self.assert_entries(koen_ldg=1)
session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
session.commit() session.commit()
entries_changed = compute_logbook_entries(session) update_logbook(session)
self.assertEqual(entries_changed, '1/0') self.assert_entries(koen_complete=1)
if __name__ == '__main__': if __name__ == '__main__':

Wyświetl plik

@ -3,7 +3,7 @@ import os
from ogn.model import TakeoffLanding from ogn.model import TakeoffLanding
from ogn.collect.takeoff_landing import compute_takeoff_and_landing from ogn.collect.takeoff_landing import update_takeoff_landing
class TestDB(unittest.TestCase): class TestDB(unittest.TestCase):
@ -30,6 +30,7 @@ class TestDB(unittest.TestCase):
session = self.session session = self.session
session.execute("DELETE FROM takeoff_landing") session.execute("DELETE FROM takeoff_landing")
session.execute("DELETE FROM aircraft_beacon") session.execute("DELETE FROM aircraft_beacon")
session.execute("DELETE FROM airport")
session.commit() session.commit()
pass pass
@ -44,6 +45,7 @@ class TestDB(unittest.TestCase):
return i return i
def test_broken_rope(self): def test_broken_rope(self):
"""Fill the db with a winch launch where the rope breaks. The algorithm should detect one takeoff and one landing."""
session = self.session session = self.session
session.execute("INSERT INTO aircraft_beacon(address, location, altitude, timestamp, track, ground_speed, climb_rate, turn_rate) VALUES('DDEFF7','0101000020E61000009668B61829F12640330E0887F1E94740',604,'2016-07-02 10:47:12',0,0,0,0)") session.execute("INSERT INTO aircraft_beacon(address, location, altitude, timestamp, track, ground_speed, climb_rate, turn_rate) VALUES('DDEFF7','0101000020E61000009668B61829F12640330E0887F1E94740',604,'2016-07-02 10:47:12',0,0,0,0)")
@ -95,7 +97,12 @@ class TestDB(unittest.TestCase):
session.execute("UPDATE aircraft_beacon SET device_id = d.id FROM device d WHERE d.address='DDEFF7'") session.execute("UPDATE aircraft_beacon SET device_id = d.id FROM device d WHERE d.address='DDEFF7'")
session.commit() session.commit()
compute_takeoff_and_landing(session) # find the takeoff and the landing
update_takeoff_landing(session)
self.assertEqual(self.count_takeoff_and_landings(), 2)
# we should not find the takeoff and the landing again
update_takeoff_landing(session)
self.assertEqual(self.count_takeoff_and_landings(), 2) self.assertEqual(self.count_takeoff_and_landings(), 2)

Wyświetl plik

@ -1,23 +0,0 @@
import unittest
import unittest.mock as mock
from ogn.gateway.manage import run
# from ogn.gateway.manage import import_logfile
class GatewayManagerTest(unittest.TestCase):
# try simple user interrupt
@mock.patch('ogn.gateway.manage.AprsClient')
def test_run_user_interruption(self, mock_aprs_client):
instance = mock_aprs_client.return_value
instance.run.side_effect = KeyboardInterrupt()
run(aprs_user="testuser")
instance.connect.assert_called_once_with()
self.assertEqual(instance.run.call_count, 1)
instance.disconnect.assert_called_once_with()
if __name__ == '__main__':
unittest.main()