kopia lustrzana https://github.com/glidernet/ogn-python
commit
df8a7346e6
16
.travis.yml
16
.travis.yml
|
@ -5,18 +5,22 @@ env:
|
||||||
|
|
||||||
python:
|
python:
|
||||||
- 3.4
|
- 3.4
|
||||||
|
- 3.5
|
||||||
|
- 3.6
|
||||||
|
|
||||||
services:
|
addons:
|
||||||
- postgresql
|
postgresql: "9.5"
|
||||||
|
apt:
|
||||||
|
packages:
|
||||||
|
- postgresql-9.5-postgis-2.3
|
||||||
|
|
||||||
before_script:
|
before_script:
|
||||||
- flake8 tests ogn
|
- flake8 tests ogn_test
|
||||||
- psql -c 'CREATE DATABASE ogn_test;' -U postgres
|
- psql -U postgres -c 'CREATE DATABASE ogn_test;'
|
||||||
- psql -c 'CREATE EXTENSION postgis;' -U postgres -d ogn_test
|
- psql -U postgres -c 'CREATE EXTENSION postgis;'
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- nosetests --with-coverage --cover-package=ogn
|
- nosetests --with-coverage --cover-package=ogn
|
||||||
|
|
||||||
- pip install . --upgrade
|
- pip install . --upgrade
|
||||||
- python -c 'import ogn'
|
- python -c 'import ogn'
|
||||||
|
|
||||||
|
|
|
@ -27,8 +27,8 @@ def close_db(signal, sender):
|
||||||
app = Celery('ogn.collect',
|
app = Celery('ogn.collect',
|
||||||
include=["ogn.collect.database",
|
include=["ogn.collect.database",
|
||||||
"ogn.collect.logbook",
|
"ogn.collect.logbook",
|
||||||
|
"ogn.collect.stats",
|
||||||
"ogn.collect.takeoff_landing",
|
"ogn.collect.takeoff_landing",
|
||||||
"ogn.collect.receiver"
|
|
||||||
])
|
])
|
||||||
|
|
||||||
app.config_from_envvar("OGN_CONFIG_MODULE")
|
app.config_from_envvar("OGN_CONFIG_MODULE")
|
||||||
|
|
|
@ -1,10 +1,13 @@
|
||||||
from celery.utils.log import get_task_logger
|
from celery.utils.log import get_task_logger
|
||||||
|
|
||||||
from ogn.model import DeviceInfo, AddressOrigin
|
from sqlalchemy import insert, distinct
|
||||||
from ogn.utils import get_ddb
|
from sqlalchemy.sql import null, and_, or_, func, not_
|
||||||
|
from sqlalchemy.sql.expression import case
|
||||||
|
|
||||||
from ogn.collect.celery import app
|
from ogn.model import DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
|
||||||
|
from ogn.utils import get_ddb, get_country_code
|
||||||
|
|
||||||
|
from .celery import app
|
||||||
|
|
||||||
logger = get_task_logger(__name__)
|
logger = get_task_logger(__name__)
|
||||||
|
|
||||||
|
@ -27,7 +30,7 @@ def import_ddb():
|
||||||
"""Import registered devices from the DDB."""
|
"""Import registered devices from the DDB."""
|
||||||
|
|
||||||
logger.info("Import registered devices fom the DDB...")
|
logger.info("Import registered devices fom the DDB...")
|
||||||
address_origin = AddressOrigin.ogn_ddb
|
address_origin = DeviceInfoOrigin.ogn_ddb
|
||||||
|
|
||||||
counter = update_device_infos(app.session, address_origin)
|
counter = update_device_infos(app.session, address_origin)
|
||||||
logger.info("Imported {} devices.".format(counter))
|
logger.info("Imported {} devices.".format(counter))
|
||||||
|
@ -38,7 +41,212 @@ def import_file(path='tests/custom_ddb.txt'):
|
||||||
"""Import registered devices from a local file."""
|
"""Import registered devices from a local file."""
|
||||||
|
|
||||||
logger.info("Import registered devices from '{}'...".format(path))
|
logger.info("Import registered devices from '{}'...".format(path))
|
||||||
address_origin = AddressOrigin.user_defined
|
address_origin = DeviceInfoOrigin.user_defined
|
||||||
|
|
||||||
counter = update_device_infos(app.session, address_origin, csvfile=path)
|
counter = update_device_infos(app.session, address_origin, csvfile=path)
|
||||||
logger.info("Imported {} devices.".format(counter))
|
logger.info("Imported {} devices.".format(counter))
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_devices():
|
||||||
|
"""Add/update entries in devices table and update foreign keys in aircraft beacons."""
|
||||||
|
|
||||||
|
# Create missing Device from AircraftBeacon
|
||||||
|
available_devices = app.session.query(Device.address) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
missing_devices_query = app.session.query(distinct(AircraftBeacon.address)) \
|
||||||
|
.filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \
|
||||||
|
.filter(~AircraftBeacon.address.in_(available_devices))
|
||||||
|
|
||||||
|
ins = insert(Device).from_select([Device.address], missing_devices_query)
|
||||||
|
res = app.session.execute(ins)
|
||||||
|
insert_count = res.rowcount
|
||||||
|
app.session.commit()
|
||||||
|
|
||||||
|
# For each address in the new beacons: get firstseen, lastseen and last values != NULL
|
||||||
|
last_valid_values = app.session.query(
|
||||||
|
distinct(AircraftBeacon.address).label('address'),
|
||||||
|
func.first_value(AircraftBeacon.timestamp)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)]))
|
||||||
|
.label('firstseen'),
|
||||||
|
func.last_value(AircraftBeacon.timestamp)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)]))
|
||||||
|
.label('lastseen'),
|
||||||
|
func.first_value(AircraftBeacon.aircraft_type)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None), (AircraftBeacon.aircraft_type != null(), AircraftBeacon.aircraft_type)]))
|
||||||
|
.label('aircraft_type'),
|
||||||
|
func.first_value(AircraftBeacon.stealth)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None), (AircraftBeacon.stealth != null(), AircraftBeacon.stealth)]))
|
||||||
|
.label('stealth'),
|
||||||
|
func.first_value(AircraftBeacon.software_version)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None), (AircraftBeacon.software_version != null(), AircraftBeacon.software_version)]))
|
||||||
|
.label('software_version'),
|
||||||
|
func.first_value(AircraftBeacon.hardware_version)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None), (AircraftBeacon.hardware_version != null(), AircraftBeacon.hardware_version)]))
|
||||||
|
.label('hardware_version'),
|
||||||
|
func.first_value(AircraftBeacon.real_address)
|
||||||
|
.over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)]))
|
||||||
|
.label('real_address')) \
|
||||||
|
.filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
update_values = app.session.query(
|
||||||
|
Device.address,
|
||||||
|
case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen),
|
||||||
|
(Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'),
|
||||||
|
case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'),
|
||||||
|
case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'),
|
||||||
|
case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'),
|
||||||
|
case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'),
|
||||||
|
case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'),
|
||||||
|
case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address),
|
||||||
|
(Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \
|
||||||
|
.filter(Device.address == last_valid_values.c.address) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
update_receivers = app.session.query(Device) \
|
||||||
|
.filter(Device.address == update_values.c.address) \
|
||||||
|
.update({
|
||||||
|
Device.firstseen: update_values.c.firstseen,
|
||||||
|
Device.lastseen: update_values.c.lastseen,
|
||||||
|
Device.aircraft_type: update_values.c.aircraft_type,
|
||||||
|
Device.stealth: update_values.c.stealth,
|
||||||
|
Device.software_version: update_values.c.software_version,
|
||||||
|
Device.hardware_version: update_values.c.hardware_version,
|
||||||
|
Device.real_address: update_values.c.real_address},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
# Update relations to aircraft beacons
|
||||||
|
upd = app.session.query(AircraftBeacon) \
|
||||||
|
.filter(AircraftBeacon.device_id == null()) \
|
||||||
|
.filter(AircraftBeacon.address == Device.address) \
|
||||||
|
.update({
|
||||||
|
AircraftBeacon.device_id: Device.id},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
app.session.commit()
|
||||||
|
logger.info("Devices: {} inserted, {} updated".format(insert_count, update_receivers))
|
||||||
|
logger.info("Updated {} AircraftBeacons".format(upd))
|
||||||
|
|
||||||
|
return "{} Devices inserted, {} Devices updated, {} AircraftBeacons updated" \
|
||||||
|
.format(insert_count, update_receivers, upd)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_receivers():
|
||||||
|
"""Add/update_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons."""
|
||||||
|
# Create missing Receiver from ReceiverBeacon
|
||||||
|
available_receivers = app.session.query(Receiver.name) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
missing_receiver_query = app.session.query(distinct(ReceiverBeacon.name)) \
|
||||||
|
.filter(ReceiverBeacon.receiver_id == null()) \
|
||||||
|
.filter(~ReceiverBeacon.name.in_(available_receivers))
|
||||||
|
|
||||||
|
ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query)
|
||||||
|
res = app.session.execute(ins)
|
||||||
|
insert_count = res.rowcount
|
||||||
|
|
||||||
|
# For each name in the new beacons: get firstseen, lastseen and last values != NULL
|
||||||
|
last_valid_values = app.session.query(
|
||||||
|
distinct(ReceiverBeacon.name).label('name'),
|
||||||
|
func.first_value(ReceiverBeacon.timestamp)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)]))
|
||||||
|
.label('firstseen'),
|
||||||
|
func.last_value(ReceiverBeacon.timestamp)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)]))
|
||||||
|
.label('lastseen'),
|
||||||
|
func.first_value(ReceiverBeacon.location_wkt)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None), (ReceiverBeacon.location_wkt != null(), ReceiverBeacon.location_wkt)]))
|
||||||
|
.label('location_wkt'),
|
||||||
|
func.first_value(ReceiverBeacon.altitude)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None), (ReceiverBeacon.altitude != null(), ReceiverBeacon.altitude)]))
|
||||||
|
.label('altitude'),
|
||||||
|
func.first_value(ReceiverBeacon.version)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None), (ReceiverBeacon.version != null(), ReceiverBeacon.version)]))
|
||||||
|
.label('version'),
|
||||||
|
func.first_value(ReceiverBeacon.platform)
|
||||||
|
.over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None), (ReceiverBeacon.platform != null(), ReceiverBeacon.platform)]))
|
||||||
|
.label('platform')) \
|
||||||
|
.filter(ReceiverBeacon.receiver_id == null()) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
update_values = app.session.query(
|
||||||
|
Receiver.name,
|
||||||
|
case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen),
|
||||||
|
(Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'),
|
||||||
|
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen),
|
||||||
|
(Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'),
|
||||||
|
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)),
|
||||||
|
(Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'),
|
||||||
|
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude),
|
||||||
|
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'),
|
||||||
|
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version),
|
||||||
|
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'),
|
||||||
|
case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform),
|
||||||
|
(Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'),
|
||||||
|
case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed
|
||||||
|
(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \
|
||||||
|
.filter(Receiver.name == last_valid_values.c.name) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
update_receivers = app.session.query(Receiver) \
|
||||||
|
.filter(Receiver.name == update_values.c.name) \
|
||||||
|
.update({
|
||||||
|
Receiver.firstseen: update_values.c.firstseen,
|
||||||
|
Receiver.lastseen: update_values.c.lastseen,
|
||||||
|
Receiver.location_wkt: update_values.c.location_wkt,
|
||||||
|
Receiver.altitude: update_values.c.altitude,
|
||||||
|
Receiver.version: update_values.c.version,
|
||||||
|
Receiver.platform: update_values.c.platform,
|
||||||
|
Receiver.country_code: update_values.c.country_code},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
# Update relations to aircraft beacons
|
||||||
|
update_aircraft_beacons = app.session.query(AircraftBeacon) \
|
||||||
|
.filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \
|
||||||
|
.update({AircraftBeacon.receiver_id: Receiver.id,
|
||||||
|
AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
# Update relations to receiver beacons
|
||||||
|
update_receiver_beacons = app.session.query(ReceiverBeacon) \
|
||||||
|
.filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \
|
||||||
|
.update({ReceiverBeacon.receiver_id: Receiver.id},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
app.session.commit()
|
||||||
|
|
||||||
|
logger.info("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers))
|
||||||
|
logger.info("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons))
|
||||||
|
|
||||||
|
return "{} Receivers inserted, {} Receivers updated, {} AircraftBeacons updated, {} ReceiverBeacons updated" \
|
||||||
|
.format(insert_count, update_receivers, update_aircraft_beacons, update_receiver_beacons)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_country_code():
|
||||||
|
# update country code in receivers table if None
|
||||||
|
unknown_country_query = app.session.query(Receiver) \
|
||||||
|
.filter(Receiver.country_code == null()) \
|
||||||
|
.filter(Receiver.location_wkt != null()) \
|
||||||
|
.order_by(Receiver.name)
|
||||||
|
|
||||||
|
counter = 0
|
||||||
|
for receiver in unknown_country_query.all():
|
||||||
|
location = receiver.location
|
||||||
|
country_code = get_country_code(location.latitude, location.longitude)
|
||||||
|
if country_code is not None:
|
||||||
|
receiver.country_code = country_code
|
||||||
|
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
|
||||||
|
counter += 1
|
||||||
|
|
||||||
|
app.session.commit()
|
||||||
|
|
||||||
|
return "Updated country_code for {} Receivers".format(counter)
|
||||||
|
|
|
@ -1,25 +1,22 @@
|
||||||
from celery.utils.log import get_task_logger
|
from celery.utils.log import get_task_logger
|
||||||
|
|
||||||
from sqlalchemy import and_, or_, insert, update, between, exists
|
from sqlalchemy import and_, or_, insert, update, exists
|
||||||
from sqlalchemy.sql import func, null
|
from sqlalchemy.sql import func, null
|
||||||
from sqlalchemy.sql.expression import true, false
|
from sqlalchemy.sql.expression import true, false
|
||||||
|
|
||||||
from ogn.collect.celery import app
|
from ogn.collect.celery import app
|
||||||
from ogn.model import TakeoffLanding, Logbook
|
from ogn.model import TakeoffLanding, Logbook, AircraftBeacon
|
||||||
|
|
||||||
logger = get_task_logger(__name__)
|
logger = get_task_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@app.task
|
@app.task
|
||||||
def compute_logbook_entries(session=None):
|
def update_logbook(session=None):
|
||||||
logger.info("Compute logbook.")
|
logger.info("Compute logbook.")
|
||||||
|
|
||||||
if session is None:
|
if session is None:
|
||||||
session = app.session
|
session = app.session
|
||||||
|
|
||||||
or_args = [between(TakeoffLanding.timestamp, '2016-06-28 00:00:00', '2016-06-28 23:59:59')]
|
|
||||||
or_args = []
|
|
||||||
|
|
||||||
# 'wo' is the window order for the sql window function
|
# 'wo' is the window order for the sql window function
|
||||||
wo = and_(func.date(TakeoffLanding.timestamp),
|
wo = and_(func.date(TakeoffLanding.timestamp),
|
||||||
TakeoffLanding.device_id,
|
TakeoffLanding.device_id,
|
||||||
|
@ -43,7 +40,6 @@ def compute_logbook_entries(session=None):
|
||||||
TakeoffLanding.airport_id,
|
TakeoffLanding.airport_id,
|
||||||
func.lag(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_prev'),
|
func.lag(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_prev'),
|
||||||
func.lead(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_next')) \
|
func.lead(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_next')) \
|
||||||
.filter(*or_args) \
|
|
||||||
.subquery()
|
.subquery()
|
||||||
|
|
||||||
# find complete flights (with takeoff and landing on the same day)
|
# find complete flights (with takeoff and landing on the same day)
|
||||||
|
@ -156,4 +152,36 @@ def compute_logbook_entries(session=None):
|
||||||
session.commit()
|
session.commit()
|
||||||
logger.debug("New logbook entries: {}".format(insert_counter))
|
logger.debug("New logbook entries: {}".format(insert_counter))
|
||||||
|
|
||||||
return "{}/{}".format(update_counter, insert_counter)
|
return "Logbook entries: {} inserted, {} updated".format(update_counter, insert_counter)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_max_altitude(session=None):
|
||||||
|
logger.info("Update logbook max altitude.")
|
||||||
|
|
||||||
|
if session is None:
|
||||||
|
session = app.session
|
||||||
|
|
||||||
|
logbook_entries = session.query(Logbook.id) \
|
||||||
|
.filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null())) \
|
||||||
|
.limit(1000) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
max_altitudes = session.query(Logbook.id, func.max(AircraftBeacon.altitude).label('max_altitude')) \
|
||||||
|
.filter(Logbook.id == logbook_entries.c.id) \
|
||||||
|
.filter(and_(AircraftBeacon.device_id == Logbook.device_id,
|
||||||
|
AircraftBeacon.timestamp >= Logbook.takeoff_timestamp,
|
||||||
|
AircraftBeacon.timestamp <= Logbook.landing_timestamp)) \
|
||||||
|
.group_by(Logbook.id) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
update_logbook = app.session.query(Logbook) \
|
||||||
|
.filter(Logbook.id == max_altitudes.c.id) \
|
||||||
|
.update({
|
||||||
|
Logbook.max_altitude: max_altitudes.c.max_altitude},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
|
session.commit()
|
||||||
|
logger.info("Logbook: {} entries updated.".format(update_logbook))
|
||||||
|
|
||||||
|
return "Logbook: {} entries updated.".format(update_logbook)
|
||||||
|
|
|
@ -1,111 +0,0 @@
|
||||||
from sqlalchemy.sql import func, null
|
|
||||||
from sqlalchemy.sql.functions import coalesce
|
|
||||||
from sqlalchemy import and_, not_, or_
|
|
||||||
|
|
||||||
from celery.utils.log import get_task_logger
|
|
||||||
|
|
||||||
from ogn.model import Receiver, ReceiverBeacon
|
|
||||||
from ogn.utils import get_country_code
|
|
||||||
from ogn.collect.celery import app
|
|
||||||
|
|
||||||
logger = get_task_logger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
@app.task
|
|
||||||
def update_receivers():
|
|
||||||
"""Update the receiver table."""
|
|
||||||
# get the timestamp of last update
|
|
||||||
last_update_query = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry'))
|
|
||||||
last_update = last_update_query.one().last_entry
|
|
||||||
|
|
||||||
# get last receiver beacons since last update
|
|
||||||
last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name,
|
|
||||||
func.max(ReceiverBeacon.timestamp).label('lastseen')) \
|
|
||||||
.filter(ReceiverBeacon.timestamp >= last_update) \
|
|
||||||
.group_by(ReceiverBeacon.name) \
|
|
||||||
.subquery()
|
|
||||||
|
|
||||||
# update receivers
|
|
||||||
receivers_to_update = app.session.query(ReceiverBeacon.name,
|
|
||||||
ReceiverBeacon.location_wkt,
|
|
||||||
ReceiverBeacon.altitude,
|
|
||||||
last_receiver_beacon_sq.columns.lastseen,
|
|
||||||
ReceiverBeacon.version,
|
|
||||||
ReceiverBeacon.platform) \
|
|
||||||
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
|
|
||||||
ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
|
|
||||||
.subquery()
|
|
||||||
|
|
||||||
# ... set country code to None if lat or lon changed
|
|
||||||
changed_count = app.session.query(Receiver) \
|
|
||||||
.filter(Receiver.name == receivers_to_update.columns.name) \
|
|
||||||
.filter(or_(not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)),
|
|
||||||
and_(Receiver.location_wkt == null(),
|
|
||||||
receivers_to_update.columns.location != null()))) \
|
|
||||||
.update({"location_wkt": receivers_to_update.columns.location,
|
|
||||||
"country_code": null()},
|
|
||||||
synchronize_session=False)
|
|
||||||
|
|
||||||
# ... and update altitude, lastseen, version and platform
|
|
||||||
update_count = app.session.query(Receiver) \
|
|
||||||
.filter(Receiver.name == receivers_to_update.columns.name) \
|
|
||||||
.update({"altitude": receivers_to_update.columns.altitude,
|
|
||||||
"lastseen": receivers_to_update.columns.lastseen,
|
|
||||||
"version": receivers_to_update.columns.version,
|
|
||||||
"platform": receivers_to_update.columns.platform})
|
|
||||||
|
|
||||||
# add new receivers
|
|
||||||
empty_sq = app.session.query(ReceiverBeacon.name,
|
|
||||||
ReceiverBeacon.location_wkt,
|
|
||||||
ReceiverBeacon.altitude,
|
|
||||||
last_receiver_beacon_sq.columns.lastseen,
|
|
||||||
ReceiverBeacon.version, ReceiverBeacon.platform) \
|
|
||||||
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
|
|
||||||
ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
|
|
||||||
.outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \
|
|
||||||
.filter(Receiver.name == null()) \
|
|
||||||
.order_by(ReceiverBeacon.name)
|
|
||||||
|
|
||||||
for receiver_beacon in empty_sq.all():
|
|
||||||
receiver = Receiver()
|
|
||||||
receiver.name = receiver_beacon.name
|
|
||||||
receiver.location_wkt = receiver_beacon.location_wkt
|
|
||||||
receiver.altitude = receiver_beacon.altitude
|
|
||||||
receiver.firstseen = None
|
|
||||||
receiver.lastseen = receiver_beacon.lastseen
|
|
||||||
receiver.version = receiver_beacon.version
|
|
||||||
receiver.platform = receiver_beacon.platform
|
|
||||||
|
|
||||||
app.session.add(receiver)
|
|
||||||
logger.info("{} added".format(receiver.name))
|
|
||||||
|
|
||||||
# update firstseen if None
|
|
||||||
firstseen_null_query = app.session.query(Receiver.name,
|
|
||||||
func.min(ReceiverBeacon.timestamp).label('firstseen')) \
|
|
||||||
.filter(Receiver.firstseen == null()) \
|
|
||||||
.join(ReceiverBeacon, Receiver.name == ReceiverBeacon.name) \
|
|
||||||
.group_by(Receiver.name) \
|
|
||||||
.subquery()
|
|
||||||
|
|
||||||
added_count = app.session.query(Receiver) \
|
|
||||||
.filter(Receiver.name == firstseen_null_query.columns.name) \
|
|
||||||
.update({'firstseen': firstseen_null_query.columns.firstseen})
|
|
||||||
|
|
||||||
# update country code if None
|
|
||||||
unknown_country_query = app.session.query(Receiver) \
|
|
||||||
.filter(Receiver.country_code == null()) \
|
|
||||||
.filter(Receiver.location_wkt != null()) \
|
|
||||||
.order_by(Receiver.name)
|
|
||||||
|
|
||||||
for receiver in unknown_country_query.all():
|
|
||||||
location = receiver.location
|
|
||||||
country_code = get_country_code(location.latitude, location.longitude)
|
|
||||||
if country_code is not None:
|
|
||||||
receiver.country_code = country_code
|
|
||||||
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
|
|
||||||
|
|
||||||
logger.info("Added: {}, location changed: {}".format(added_count, changed_count))
|
|
||||||
|
|
||||||
app.session.commit()
|
|
||||||
|
|
||||||
return update_count
|
|
|
@ -0,0 +1,85 @@
|
||||||
|
from celery.utils.log import get_task_logger
|
||||||
|
|
||||||
|
from sqlalchemy import insert, distinct
|
||||||
|
from sqlalchemy.sql import null, and_, func
|
||||||
|
from sqlalchemy.sql.expression import literal_column
|
||||||
|
|
||||||
|
from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats
|
||||||
|
|
||||||
|
from .celery import app
|
||||||
|
|
||||||
|
logger = get_task_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_device_stats(date=None):
|
||||||
|
"""Add/update entries in device stats table."""
|
||||||
|
|
||||||
|
if not date:
|
||||||
|
logger.warn("A date is needed for calculating stats. Exiting")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# First kill the stats for the selected date
|
||||||
|
deleted_counter = app.session.query(DeviceStats) \
|
||||||
|
.filter(DeviceStats.date == date) \
|
||||||
|
.delete()
|
||||||
|
|
||||||
|
# Calculate stats for the selected date
|
||||||
|
device_stats = app.session.query(
|
||||||
|
AircraftBeacon.device_id,
|
||||||
|
func.date(AircraftBeacon.timestamp).label('date'),
|
||||||
|
func.count(distinct(AircraftBeacon.receiver_id)).label('receiver_count'),
|
||||||
|
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
|
||||||
|
func.max(AircraftBeacon.altitude).label('max_altitude')) \
|
||||||
|
.filter(and_(AircraftBeacon.device_id != null(), AircraftBeacon.receiver_id != null())) \
|
||||||
|
.filter(func.date(AircraftBeacon.timestamp) == date) \
|
||||||
|
.group_by(AircraftBeacon.device_id, func.date(AircraftBeacon.timestamp)) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
# And insert them
|
||||||
|
ins = insert(DeviceStats).from_select(
|
||||||
|
[DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.aircraft_beacon_count, DeviceStats.max_altitude],
|
||||||
|
device_stats)
|
||||||
|
res = app.session.execute(ins)
|
||||||
|
insert_counter = res.rowcount
|
||||||
|
app.session.commit()
|
||||||
|
logger.debug("DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
|
||||||
|
|
||||||
|
return "DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def update_receiver_stats(date=None):
|
||||||
|
"""Add/update entries in receiver stats table."""
|
||||||
|
|
||||||
|
if not date:
|
||||||
|
logger.warn("A date is needed for calculating stats. Exiting")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# First kill the stats for the selected date
|
||||||
|
deleted_counter = app.session.query(ReceiverStats) \
|
||||||
|
.filter(ReceiverStats.date == date) \
|
||||||
|
.delete()
|
||||||
|
|
||||||
|
# Calculate stats for the selected date
|
||||||
|
receiver_stats = app.session.query(
|
||||||
|
AircraftBeacon.receiver_id,
|
||||||
|
literal_column("'{}'".format(date)).label('date'),
|
||||||
|
func.count(AircraftBeacon.id).label('aircraft_beacon_count'),
|
||||||
|
func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'),
|
||||||
|
func.max(AircraftBeacon.distance).label('max_distance')) \
|
||||||
|
.filter(AircraftBeacon.receiver_id != null()) \
|
||||||
|
.filter(func.date(AircraftBeacon.timestamp) == date) \
|
||||||
|
.group_by(AircraftBeacon.receiver_id) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
|
# And insert them
|
||||||
|
ins = insert(ReceiverStats).from_select(
|
||||||
|
[ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance],
|
||||||
|
receiver_stats)
|
||||||
|
res = app.session.execute(ins)
|
||||||
|
insert_counter = res.rowcount
|
||||||
|
app.session.commit()
|
||||||
|
logger.debug("ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
|
||||||
|
|
||||||
|
return "ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
|
|
@ -13,7 +13,7 @@ logger = get_task_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@app.task
|
@app.task
|
||||||
def compute_takeoff_and_landing(session=None):
|
def update_takeoff_landing(session=None):
|
||||||
logger.info("Compute takeoffs and landings.")
|
logger.info("Compute takeoffs and landings.")
|
||||||
|
|
||||||
if session is None:
|
if session is None:
|
||||||
|
@ -29,10 +29,10 @@ def compute_takeoff_and_landing(session=None):
|
||||||
takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit
|
takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit
|
||||||
landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit
|
landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit
|
||||||
duration = 100 # the points must not exceed this duration
|
duration = 100 # the points must not exceed this duration
|
||||||
radius = 0.05 # the points must not exceed this radius (degree!) around the 2nd point
|
radius = 5000 # the points must not exceed this radius around the 2nd point
|
||||||
|
|
||||||
# takeoff / landing has to be near an airport
|
# takeoff / landing has to be near an airport
|
||||||
airport_radius = 0.025 # takeoff / landing must not exceed this radius (degree!) around the airport
|
airport_radius = 2500 # takeoff / landing must not exceed this radius around the airport
|
||||||
airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport
|
airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport
|
||||||
|
|
||||||
# 'wo' is the window order for the sql window function
|
# 'wo' is the window order for the sql window function
|
||||||
|
@ -41,8 +41,19 @@ def compute_takeoff_and_landing(session=None):
|
||||||
AircraftBeacon.receiver_id)
|
AircraftBeacon.receiver_id)
|
||||||
|
|
||||||
# make a query with current, previous and next position
|
# make a query with current, previous and next position
|
||||||
|
beacon_selection = session.query(AircraftBeacon.id) \
|
||||||
|
.filter(AircraftBeacon.status == null()) \
|
||||||
|
.order_by(AircraftBeacon.timestamp) \
|
||||||
|
.limit(1000000) \
|
||||||
|
.subquery()
|
||||||
|
|
||||||
sq = session.query(
|
sq = session.query(
|
||||||
AircraftBeacon.id,
|
AircraftBeacon.id,
|
||||||
|
func.lag(AircraftBeacon.id).over(order_by=wo).label('id_prev'),
|
||||||
|
func.lead(AircraftBeacon.id).over(order_by=wo).label('id_next'),
|
||||||
|
AircraftBeacon.device_id,
|
||||||
|
func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'),
|
||||||
|
func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next'),
|
||||||
AircraftBeacon.timestamp,
|
AircraftBeacon.timestamp,
|
||||||
func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'),
|
func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'),
|
||||||
func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'),
|
func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'),
|
||||||
|
@ -57,16 +68,14 @@ def compute_takeoff_and_landing(session=None):
|
||||||
func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'),
|
func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'),
|
||||||
AircraftBeacon.altitude,
|
AircraftBeacon.altitude,
|
||||||
func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'),
|
func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'),
|
||||||
func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next'),
|
func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next')) \
|
||||||
AircraftBeacon.device_id,
|
.filter(AircraftBeacon.id == beacon_selection.c.id) \
|
||||||
func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'),
|
|
||||||
func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next')) \
|
|
||||||
.filter(AircraftBeacon.status == null()) \
|
|
||||||
.subquery()
|
.subquery()
|
||||||
|
|
||||||
|
# consider only positions with the same device id
|
||||||
sq2 = session.query(sq) \
|
sq2 = session.query(sq) \
|
||||||
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
|
.filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \
|
||||||
.subquery()
|
.subquery()
|
||||||
|
|
||||||
# find possible takeoffs and landings
|
# find possible takeoffs and landings
|
||||||
sq3 = session.query(
|
sq3 = session.query(
|
||||||
|
@ -82,8 +91,8 @@ def compute_takeoff_and_landing(session=None):
|
||||||
(sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'),
|
(sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'),
|
||||||
sq2.c.device_id) \
|
sq2.c.device_id) \
|
||||||
.filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \
|
.filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \
|
||||||
.filter(and_(func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_prev, radius),
|
.filter(and_(func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_prev) < radius,
|
||||||
func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_next, radius))) \
|
func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_next) < radius)) \
|
||||||
.filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff
|
.filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff
|
||||||
sq2.c.ground_speed > takeoff_speed,
|
sq2.c.ground_speed > takeoff_speed,
|
||||||
sq2.c.ground_speed_next > takeoff_speed),
|
sq2.c.ground_speed_next > takeoff_speed),
|
||||||
|
@ -99,7 +108,7 @@ def compute_takeoff_and_landing(session=None):
|
||||||
sq3.c.is_takeoff,
|
sq3.c.is_takeoff,
|
||||||
sq3.c.device_id,
|
sq3.c.device_id,
|
||||||
Airport.id.label('airport_id')) \
|
Airport.id.label('airport_id')) \
|
||||||
.filter(and_(func.ST_DFullyWithin(sq3.c.location, Airport.location_wkt, airport_radius),
|
.filter(and_(func.ST_Distance_Sphere(sq3.c.location, Airport.location_wkt) < airport_radius,
|
||||||
between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \
|
between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \
|
||||||
.filter(between(Airport.style, 2, 5)) \
|
.filter(between(Airport.style, 2, 5)) \
|
||||||
.subquery()
|
.subquery()
|
||||||
|
@ -120,7 +129,14 @@ def compute_takeoff_and_landing(session=None):
|
||||||
takeoff_landing_query)
|
takeoff_landing_query)
|
||||||
result = session.execute(ins)
|
result = session.execute(ins)
|
||||||
counter = result.rowcount
|
counter = result.rowcount
|
||||||
|
|
||||||
|
# mark the computated AircraftBeacons as 'used'
|
||||||
|
update_aircraft_beacons = session.query(AircraftBeacon) \
|
||||||
|
.filter(AircraftBeacon.id == sq2.c.id) \
|
||||||
|
.update({AircraftBeacon.status: 1},
|
||||||
|
synchronize_session='fetch')
|
||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
logger.debug("New takeoffs and landings: {}".format(counter))
|
logger.debug("Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(counter, update_aircraft_beacons))
|
||||||
|
|
||||||
return counter
|
return counter
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
from manager import Manager
|
from manager import Manager
|
||||||
from ogn.commands.dbutils import session
|
from ogn.commands.dbutils import session
|
||||||
|
@ -9,18 +10,19 @@ from ogn.utils import open_file
|
||||||
|
|
||||||
manager = Manager()
|
manager = Manager()
|
||||||
|
|
||||||
|
PATTERN = '^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$'
|
||||||
|
|
||||||
|
|
||||||
@manager.command
|
@manager.command
|
||||||
def convert_logfile(path, logfile='main.log', loglevel='INFO'):
|
def convert_logfile(path, logfile='main.log', loglevel='INFO'):
|
||||||
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
|
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
|
||||||
|
|
||||||
if os.path.isfile(path):
|
if os.path.isfile(path):
|
||||||
print("Reading file: {}".format(path))
|
head, tail = os.path.split(path)
|
||||||
convert(path)
|
convert(tail, path=head)
|
||||||
print("Finished")
|
print("Finished")
|
||||||
elif os.path.isdir(path):
|
elif os.path.isdir(path):
|
||||||
for filename in os.listdir(path):
|
for filename in os.listdir(path):
|
||||||
print("Reading file: {}".format(filename))
|
|
||||||
convert(filename, path=path)
|
convert(filename, path=path)
|
||||||
print("Finished")
|
print("Finished")
|
||||||
else:
|
else:
|
||||||
|
@ -28,15 +30,25 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'):
|
||||||
|
|
||||||
|
|
||||||
def convert(sourcefile, path=''):
|
def convert(sourcefile, path=''):
|
||||||
import re
|
|
||||||
import csv
|
import csv
|
||||||
import gzip
|
import gzip
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile)
|
match = re.search(PATTERN, sourcefile)
|
||||||
if match:
|
if match:
|
||||||
reference_date_string = match.group(1)
|
reference_date_string = match.group(1)
|
||||||
reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d")
|
reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d")
|
||||||
|
|
||||||
|
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
|
||||||
|
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
|
||||||
|
|
||||||
|
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
|
||||||
|
print("Reading file: {}".format(sourcefile))
|
||||||
|
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
|
||||||
|
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
|
||||||
|
else:
|
||||||
|
print("Output files for file {} already exists. Skipping".format(sourcefile))
|
||||||
|
return
|
||||||
else:
|
else:
|
||||||
print("filename '{}' does not match pattern. Skipping".format(sourcefile))
|
print("filename '{}' does not match pattern. Skipping".format(sourcefile))
|
||||||
return
|
return
|
||||||
|
@ -49,16 +61,6 @@ def convert(sourcefile, path=''):
|
||||||
total += 1
|
total += 1
|
||||||
fin.seek(0)
|
fin.seek(0)
|
||||||
|
|
||||||
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
|
|
||||||
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
|
|
||||||
|
|
||||||
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
|
|
||||||
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
|
|
||||||
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
|
|
||||||
else:
|
|
||||||
print("Output files already exists. Skipping")
|
|
||||||
return
|
|
||||||
|
|
||||||
aircraft_beacons = list()
|
aircraft_beacons = list()
|
||||||
receiver_beacons = list()
|
receiver_beacons = list()
|
||||||
|
|
||||||
|
@ -182,7 +184,7 @@ def import_logfile(path):
|
||||||
else:
|
else:
|
||||||
print("For {} beacons already exist. Skipping".format(reference_date_string))
|
print("For {} beacons already exist. Skipping".format(reference_date_string))
|
||||||
else:
|
else:
|
||||||
print("Unknown file type: {}".format())
|
print("Unknown file type: {}".format(tail))
|
||||||
|
|
||||||
|
|
||||||
def check_no_beacons(tablename, reference_date_string):
|
def check_no_beacons(tablename, reference_date_string):
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
from manager import Manager
|
from manager import Manager
|
||||||
from ogn.collect.database import update_device_infos
|
from ogn.collect.database import update_device_infos
|
||||||
from ogn.commands.dbutils import engine, session
|
from ogn.commands.dbutils import engine, session
|
||||||
from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
|
from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon
|
||||||
from ogn.utils import get_airports
|
from ogn.utils import get_airports
|
||||||
from sqlalchemy import insert, distinct
|
from sqlalchemy import distinct
|
||||||
from sqlalchemy.sql import null
|
from sqlalchemy.sql import null, func
|
||||||
|
|
||||||
|
|
||||||
manager = Manager()
|
manager = Manager()
|
||||||
|
@ -53,7 +53,7 @@ def import_ddb():
|
||||||
"""Import registered devices from the DDB."""
|
"""Import registered devices from the DDB."""
|
||||||
|
|
||||||
print("Import registered devices fom the DDB...")
|
print("Import registered devices fom the DDB...")
|
||||||
address_origin = AddressOrigin.ogn_ddb
|
address_origin = DeviceInfoOrigin.ogn_ddb
|
||||||
counter = update_device_infos(session,
|
counter = update_device_infos(session,
|
||||||
address_origin)
|
address_origin)
|
||||||
print("Imported %i devices." % counter)
|
print("Imported %i devices." % counter)
|
||||||
|
@ -65,7 +65,7 @@ def import_file(path='tests/custom_ddb.txt'):
|
||||||
# (flushes previously manually imported entries)
|
# (flushes previously manually imported entries)
|
||||||
|
|
||||||
print("Import registered devices from '{}'...".format(path))
|
print("Import registered devices from '{}'...".format(path))
|
||||||
address_origin = AddressOrigin.user_defined
|
address_origin = DeviceInfoOrigin.user_defined
|
||||||
counter = update_device_infos(session,
|
counter = update_device_infos(session,
|
||||||
address_origin,
|
address_origin,
|
||||||
csvfile=path)
|
csvfile=path)
|
||||||
|
@ -81,53 +81,3 @@ def import_airports(path='tests/SeeYou.cup'):
|
||||||
session.bulk_save_objects(airports)
|
session.bulk_save_objects(airports)
|
||||||
session.commit()
|
session.commit()
|
||||||
print("Imported {} airports.".format(len(airports)))
|
print("Imported {} airports.".format(len(airports)))
|
||||||
|
|
||||||
|
|
||||||
@manager.command
|
|
||||||
def update_relations():
|
|
||||||
"""Update AircraftBeacon and ReceiverBeacon relations"""
|
|
||||||
|
|
||||||
# Create missing Receiver from ReceiverBeacon
|
|
||||||
available_receivers = session.query(Receiver.name) \
|
|
||||||
.subquery()
|
|
||||||
|
|
||||||
missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \
|
|
||||||
.filter(ReceiverBeacon.receiver_id == null()) \
|
|
||||||
.filter(~ReceiverBeacon.name.in_(available_receivers))
|
|
||||||
|
|
||||||
ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query)
|
|
||||||
session.execute(ins)
|
|
||||||
|
|
||||||
# Create missing Device from AircraftBeacon
|
|
||||||
available_addresses = session.query(Device.address) \
|
|
||||||
.subquery()
|
|
||||||
|
|
||||||
missing_addresses_query = session.query(distinct(AircraftBeacon.address)) \
|
|
||||||
.filter(AircraftBeacon.device_id == null()) \
|
|
||||||
.filter(~AircraftBeacon.address.in_(available_addresses))
|
|
||||||
|
|
||||||
ins2 = insert(Device).from_select([Device.address], missing_addresses_query)
|
|
||||||
session.execute(ins2)
|
|
||||||
session.commit()
|
|
||||||
print("Inserted {} Receivers and {} Devices".format(ins, ins2))
|
|
||||||
return
|
|
||||||
|
|
||||||
# Update AircraftBeacons
|
|
||||||
upd = session.query(AircraftBeacon) \
|
|
||||||
.filter(AircraftBeacon.device_id == null()) \
|
|
||||||
.filter(AircraftBeacon.receiver_id == null()) \
|
|
||||||
.filter(AircraftBeacon.address == Device.address) \
|
|
||||||
.filter(AircraftBeacon.receiver_name == Receiver.name) \
|
|
||||||
.update({AircraftBeacon.device_id: Device.id,
|
|
||||||
AircraftBeacon.receiver_id: Receiver.id},
|
|
||||||
synchronize_session='fetch')
|
|
||||||
|
|
||||||
upd2 = session.query(ReceiverBeacon) \
|
|
||||||
.filter(ReceiverBeacon.receiver_id == null()) \
|
|
||||||
.filter(ReceiverBeacon.receiver_name == Receiver.name) \
|
|
||||||
.update({Receiver.name: ReceiverBeacon.receiver_name},
|
|
||||||
synchronize_session='fetch')
|
|
||||||
|
|
||||||
session.commit()
|
|
||||||
print("Updated {} AircraftBeacons and {} ReceiverBeacons".
|
|
||||||
format(upd, upd2))
|
|
||||||
|
|
|
@ -3,8 +3,8 @@
|
||||||
from datetime import timedelta, datetime
|
from datetime import timedelta, datetime
|
||||||
|
|
||||||
from manager import Manager
|
from manager import Manager
|
||||||
from ogn.collect.logbook import compute_logbook_entries
|
from ogn.collect.logbook import update_logbook
|
||||||
from ogn.collect.takeoff_landing import compute_takeoff_and_landing
|
from ogn.collect.takeoff_landing import update_takeoff_landing
|
||||||
from ogn.commands.dbutils import session
|
from ogn.commands.dbutils import session
|
||||||
from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook
|
from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook
|
||||||
from sqlalchemy import and_, or_
|
from sqlalchemy import and_, or_
|
||||||
|
@ -19,7 +19,7 @@ manager = Manager()
|
||||||
def compute_takeoff_landing():
|
def compute_takeoff_landing():
|
||||||
"""Compute takeoffs and landings."""
|
"""Compute takeoffs and landings."""
|
||||||
print("Compute takeoffs and landings...")
|
print("Compute takeoffs and landings...")
|
||||||
result = compute_takeoff_and_landing.delay()
|
result = update_takeoff_landing.delay()
|
||||||
counter = result.get()
|
counter = result.get()
|
||||||
print("New takeoffs/landings: {}".format(counter))
|
print("New takeoffs/landings: {}".format(counter))
|
||||||
|
|
||||||
|
@ -28,7 +28,7 @@ def compute_takeoff_landing():
|
||||||
def compute_logbook():
|
def compute_logbook():
|
||||||
"""Compute logbook."""
|
"""Compute logbook."""
|
||||||
print("Compute logbook...")
|
print("Compute logbook...")
|
||||||
result = compute_logbook_entries.delay()
|
result = update_logbook.delay()
|
||||||
counter = result.get()
|
counter = result.get()
|
||||||
print("New logbook entries: {}".format(counter))
|
print("New logbook entries: {}".format(counter))
|
||||||
|
|
||||||
|
@ -63,7 +63,8 @@ def show(airport_name, utc_delta_hours=0, date=None):
|
||||||
# get all logbook entries and add device and airport infos
|
# get all logbook entries and add device and airport infos
|
||||||
takeoff_airport = aliased(Airport, name='takeoff_airport')
|
takeoff_airport = aliased(Airport, name='takeoff_airport')
|
||||||
landing_airport = aliased(Airport, name='landing_airport')
|
landing_airport = aliased(Airport, name='landing_airport')
|
||||||
logbook_query = session.query(Logbook,
|
logbook_query = session.query(func.row_number().over(order_by=Logbook.reftime).label('row_number'),
|
||||||
|
Logbook,
|
||||||
Device,
|
Device,
|
||||||
sq3.c.registration,
|
sq3.c.registration,
|
||||||
sq3.c.aircraft) \
|
sq3.c.aircraft) \
|
||||||
|
@ -105,8 +106,9 @@ def show(airport_name, utc_delta_hours=0, date=None):
|
||||||
def none_altitude_replacer(altitude_object, airport_object):
|
def none_altitude_replacer(altitude_object, airport_object):
|
||||||
return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude)
|
return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude)
|
||||||
|
|
||||||
for [logbook, device, registration, aircraft] in logbook_query.all():
|
for [row_number, logbook, device, registration, aircraft] in logbook_query.all():
|
||||||
print('%10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % (
|
print('%3d. %10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % (
|
||||||
|
row_number,
|
||||||
logbook.reftime.date(),
|
logbook.reftime.date(),
|
||||||
none_datetime_replacer(logbook.takeoff_timestamp),
|
none_datetime_replacer(logbook.takeoff_timestamp),
|
||||||
none_track_replacer(logbook.takeoff_track),
|
none_track_replacer(logbook.takeoff_track),
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from manager import Manager
|
from manager import Manager
|
||||||
from ogn.commands.dbutils import session
|
from ogn.commands.dbutils import session
|
||||||
from ogn.model import AddressOrigin
|
from ogn.model import DeviceInfoOrigin
|
||||||
from ogn.model.device_info import DeviceInfo
|
from ogn.model.device_info import DeviceInfo
|
||||||
from sqlalchemy import func, and_, true, false
|
from sqlalchemy import func, and_, true, false
|
||||||
|
|
||||||
|
@ -39,7 +39,7 @@ def get_devices_stats(session):
|
||||||
|
|
||||||
stats = {}
|
stats = {}
|
||||||
for [address_origin, device_count, default_count, nt_count, ni_count, ntni_count] in query.all():
|
for [address_origin, device_count, default_count, nt_count, ni_count, ntni_count] in query.all():
|
||||||
origin = AddressOrigin(address_origin).name()
|
origin = DeviceInfoOrigin(address_origin).name()
|
||||||
stats[origin] = {'device_count': device_count,
|
stats[origin] = {'device_count': device_count,
|
||||||
'default_count': default_count,
|
'default_count': default_count,
|
||||||
'nt_count': nt_count,
|
'nt_count': nt_count,
|
||||||
|
|
|
@ -31,6 +31,9 @@ def message_to_beacon(raw_message, reference_date):
|
||||||
beacon = ReceiverBeacon(**message)
|
beacon = ReceiverBeacon(**message)
|
||||||
else:
|
else:
|
||||||
print("Whoops: what is this: {}".format(message))
|
print("Whoops: what is this: {}".format(message))
|
||||||
|
except NotImplementedError as e:
|
||||||
|
logger.error('Received message: {}'.format(raw_message))
|
||||||
|
logger.error(e)
|
||||||
except ParseError as e:
|
except ParseError as e:
|
||||||
logger.error('Received message: {}'.format(raw_message))
|
logger.error('Received message: {}'.format(raw_message))
|
||||||
logger.error('Drop packet, {}'.format(e.message))
|
logger.error('Drop packet, {}'.format(e.message))
|
||||||
|
|
|
@ -1,13 +1,15 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from .address_origin import AddressOrigin
|
|
||||||
from .aircraft_type import AircraftType
|
from .aircraft_type import AircraftType
|
||||||
from .base import Base
|
from .base import Base
|
||||||
from .beacon import Beacon
|
from .beacon import Beacon
|
||||||
from .device import Device
|
from .device import Device
|
||||||
from .device_info import DeviceInfo
|
from .device_info import DeviceInfo
|
||||||
|
from .device_info_origin import DeviceInfoOrigin
|
||||||
|
from .device_stats import DeviceStats
|
||||||
from .aircraft_beacon import AircraftBeacon
|
from .aircraft_beacon import AircraftBeacon
|
||||||
from .receiver_beacon import ReceiverBeacon
|
from .receiver_beacon import ReceiverBeacon
|
||||||
from .receiver import Receiver
|
from .receiver import Receiver
|
||||||
|
from .receiver_stats import ReceiverStats
|
||||||
from .takeoff_landing import TakeoffLanding
|
from .takeoff_landing import TakeoffLanding
|
||||||
from .airport import Airport
|
from .airport import Airport
|
||||||
from .logbook import Logbook
|
from .logbook import Logbook
|
||||||
|
|
|
@ -38,6 +38,9 @@ class AircraftBeacon(Beacon):
|
||||||
|
|
||||||
status = Column(SmallInteger, index=True)
|
status = Column(SmallInteger, index=True)
|
||||||
|
|
||||||
|
# Calculated values
|
||||||
|
distance = Column(Float)
|
||||||
|
|
||||||
# Relations
|
# Relations
|
||||||
receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)
|
receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)
|
||||||
receiver = relationship('Receiver', foreign_keys=[receiver_id])
|
receiver = relationship('Receiver', foreign_keys=[receiver_id])
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger
|
from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger, DateTime
|
||||||
from sqlalchemy.orm import relationship
|
from sqlalchemy.orm import relationship
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
|
@ -9,11 +9,15 @@ class Device(Base):
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True)
|
id = Column(Integer, primary_key=True)
|
||||||
address = Column(String(6), index=True)
|
address = Column(String(6), index=True)
|
||||||
|
firstseen = Column(DateTime, index=True)
|
||||||
|
lastseen = Column(DateTime, index=True)
|
||||||
aircraft_type = Column(SmallInteger, index=True)
|
aircraft_type = Column(SmallInteger, index=True)
|
||||||
stealth = Column(Boolean)
|
stealth = Column(Boolean)
|
||||||
software_version = Column(Float)
|
software_version = Column(Float)
|
||||||
hardware_version = Column(SmallInteger)
|
hardware_version = Column(SmallInteger)
|
||||||
real_address = Column(String(6))
|
real_address = Column(String(6))
|
||||||
|
firstseen = Column(DateTime, index=True)
|
||||||
|
lastseen = Column(DateTime, index=True)
|
||||||
|
|
||||||
# Relations
|
# Relations
|
||||||
aircraft_beacons = relationship('AircraftBeacon')
|
aircraft_beacons = relationship('AircraftBeacon')
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
class AddressOrigin:
|
class DeviceInfoOrigin:
|
||||||
unknown = 0
|
unknown = 0
|
||||||
ogn_ddb = 1
|
ogn_ddb = 1
|
||||||
flarmnet = 2
|
flarmnet = 2
|
|
@ -0,0 +1,19 @@
|
||||||
|
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DeviceStats(Base):
|
||||||
|
__tablename__ = "device_stats"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
|
||||||
|
date = Column(Date)
|
||||||
|
receiver_count = Column(Integer)
|
||||||
|
aircraft_beacon_count = Column(Integer)
|
||||||
|
max_altitude = Column(Float)
|
||||||
|
|
||||||
|
# Relations
|
||||||
|
device_id = Column(Integer, ForeignKey('device.id', ondelete='SET NULL'), index=True)
|
||||||
|
device = relationship('Device', foreign_keys=[device_id])
|
|
@ -0,0 +1,20 @@
|
||||||
|
from sqlalchemy import Column, Integer, Date, Float, ForeignKey
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ReceiverStats(Base):
|
||||||
|
__tablename__ = "receiver_stats"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
|
||||||
|
date = Column(Date)
|
||||||
|
aircraft_beacon_count = Column(Integer)
|
||||||
|
receiver_beacon_count = Column(Integer)
|
||||||
|
aircraft_count = Column(Integer)
|
||||||
|
max_distance = Column(Float)
|
||||||
|
|
||||||
|
# Relations
|
||||||
|
receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)
|
||||||
|
receiver = relationship('Receiver', foreign_keys=[receiver_id])
|
|
@ -8,7 +8,7 @@ from geopy.geocoders import Nominatim
|
||||||
from ogn.parser.utils import feet2m
|
from ogn.parser.utils import feet2m
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from .model import AddressOrigin, DeviceInfo, Airport, Location
|
from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location
|
||||||
|
|
||||||
|
|
||||||
DDB_URL = "http://ddb.glidernet.org/download/?t=1"
|
DDB_URL = "http://ddb.glidernet.org/download/?t=1"
|
||||||
|
@ -22,7 +22,7 @@ nm2m = 1852
|
||||||
mi2m = 1609.34
|
mi2m = 1609.34
|
||||||
|
|
||||||
|
|
||||||
def get_ddb(csvfile=None, address_origin=AddressOrigin.unknown):
|
def get_ddb(csvfile=None, address_origin=DeviceInfoOrigin.unknown):
|
||||||
if csvfile is None:
|
if csvfile is None:
|
||||||
r = requests.get(DDB_URL)
|
r = requests.get(DDB_URL)
|
||||||
rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
|
rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
|
||||||
|
|
14
setup.py
14
setup.py
|
@ -32,22 +32,22 @@ setup(
|
||||||
keywords='gliding ogn',
|
keywords='gliding ogn',
|
||||||
packages=find_packages(exclude=['tests', 'tests.*']),
|
packages=find_packages(exclude=['tests', 'tests.*']),
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'SQLAlchemy==1.1.10',
|
'SQLAlchemy==1.1.15',
|
||||||
'geopy==1.11.0',
|
'geopy==1.11.0',
|
||||||
'manage.py==0.2.10',
|
'manage.py==0.2.10',
|
||||||
'celery[redis]>=3.1,<3.2',
|
'celery[redis]>=3.1,<3.2',
|
||||||
'alembic==0.9.2',
|
'alembic==0.9.6',
|
||||||
'aerofiles==0.3',
|
'aerofiles==0.4',
|
||||||
'geoalchemy2==0.4.0',
|
'geoalchemy2==0.4.0',
|
||||||
'shapely==1.5.17.post1',
|
'shapely>=1.5.17,<1.6',
|
||||||
'ogn-client==0.8.0',
|
'ogn-client==0.8.0',
|
||||||
'psycopg2==2.7.1'
|
'psycopg2==2.7.3.2'
|
||||||
],
|
],
|
||||||
extras_require={
|
extras_require={
|
||||||
'dev': [
|
'dev': [
|
||||||
'nose==1.3.7',
|
'nose==1.3.7',
|
||||||
'coveralls==1.1',
|
'coveralls==1.2',
|
||||||
'flake8==3.3.0'
|
'flake8==3.5.0'
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
zip_safe=False
|
zip_safe=False
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
import unittest
|
import unittest
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from ogn.collect.logbook import compute_logbook_entries
|
from sqlalchemy.sql import null, and_
|
||||||
|
|
||||||
|
from ogn.model import Logbook, Airport
|
||||||
|
from ogn.collect.logbook import update_logbook
|
||||||
|
|
||||||
|
|
||||||
class TestDB(unittest.TestCase):
|
class TestDB(unittest.TestCase):
|
||||||
|
@ -35,17 +38,43 @@ class TestDB(unittest.TestCase):
|
||||||
session.commit()
|
session.commit()
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def count_logbook_entries(self):
|
||||||
|
session = self.session
|
||||||
|
query = session.query(Logbook)
|
||||||
|
return len(query.all())
|
||||||
|
|
||||||
|
def assert_entries(self, koen_to=0, koen_ldg=0, koen_complete=0, ohl_to=0, ohl_ldg=0, ohl_complete=0):
|
||||||
|
session = self.session
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.landing_airport_id == null()).all())
|
||||||
|
self.assertEqual(entries, koen_to)
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == null()).all())
|
||||||
|
self.assertEqual(entries, koen_ldg)
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all())
|
||||||
|
self.assertEqual(entries, koen_complete)
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.landing_airport_id == null()).all())
|
||||||
|
self.assertEqual(entries, ohl_to)
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == null()).all())
|
||||||
|
self.assertEqual(entries, ohl_ldg)
|
||||||
|
|
||||||
|
entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all())
|
||||||
|
self.assertEqual(entries, ohl_complete)
|
||||||
|
|
||||||
def test_single_takeoff(self):
|
def test_single_takeoff(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
|
||||||
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_to=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_to=1)
|
||||||
|
|
||||||
def test_single_landing(self):
|
def test_single_landing(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -53,11 +82,11 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_ldg=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_ldg=1)
|
||||||
|
|
||||||
def test_different_takeoffs(self):
|
def test_different_takeoffs(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -66,11 +95,11 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.TAKEOFF_OHLSTADT_DD4711)
|
session.execute(self.TAKEOFF_OHLSTADT_DD4711)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/2')
|
self.assert_entries(koen_to=1, ohl_to=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_to=1, ohl_to=1)
|
||||||
|
|
||||||
def test_takeoff_and_landing(self):
|
def test_takeoff_and_landing(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -79,11 +108,11 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_complete=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_complete=1)
|
||||||
|
|
||||||
def test_takeoff_and_landing_on_different_days(self):
|
def test_takeoff_and_landing_on_different_days(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -92,11 +121,11 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.LANDING_KOENIGSDF_DD0815_LATER)
|
session.execute(self.LANDING_KOENIGSDF_DD0815_LATER)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/2')
|
self.assert_entries(koen_to=1, koen_ldg=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_to=1, koen_ldg=1)
|
||||||
|
|
||||||
def test_update(self):
|
def test_update(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -104,23 +133,23 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_to=1)
|
||||||
|
|
||||||
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '1/0')
|
self.assert_entries(koen_complete=1)
|
||||||
|
|
||||||
session.execute(self.TAKEOFF_OHLSTADT_DD4711)
|
session.execute(self.TAKEOFF_OHLSTADT_DD4711)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_complete=1, ohl_to=1)
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/0')
|
self.assert_entries(koen_complete=1, ohl_to=1)
|
||||||
|
|
||||||
def test_update_wrong_order(self):
|
def test_update_wrong_order(self):
|
||||||
session = self.session
|
session = self.session
|
||||||
|
@ -128,14 +157,14 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
session.execute(self.LANDING_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '0/1')
|
self.assert_entries(koen_ldg=1)
|
||||||
|
|
||||||
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
session.execute(self.TAKEOFF_KOENIGSDF_DD0815)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
entries_changed = compute_logbook_entries(session)
|
update_logbook(session)
|
||||||
self.assertEqual(entries_changed, '1/0')
|
self.assert_entries(koen_complete=1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -3,7 +3,7 @@ import os
|
||||||
|
|
||||||
from ogn.model import TakeoffLanding
|
from ogn.model import TakeoffLanding
|
||||||
|
|
||||||
from ogn.collect.takeoff_landing import compute_takeoff_and_landing
|
from ogn.collect.takeoff_landing import update_takeoff_landing
|
||||||
|
|
||||||
|
|
||||||
class TestDB(unittest.TestCase):
|
class TestDB(unittest.TestCase):
|
||||||
|
@ -30,6 +30,7 @@ class TestDB(unittest.TestCase):
|
||||||
session = self.session
|
session = self.session
|
||||||
session.execute("DELETE FROM takeoff_landing")
|
session.execute("DELETE FROM takeoff_landing")
|
||||||
session.execute("DELETE FROM aircraft_beacon")
|
session.execute("DELETE FROM aircraft_beacon")
|
||||||
|
session.execute("DELETE FROM airport")
|
||||||
session.commit()
|
session.commit()
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -44,6 +45,7 @@ class TestDB(unittest.TestCase):
|
||||||
return i
|
return i
|
||||||
|
|
||||||
def test_broken_rope(self):
|
def test_broken_rope(self):
|
||||||
|
"""Fill the db with a winch launch where the rope breaks. The algorithm should detect one takeoff and one landing."""
|
||||||
session = self.session
|
session = self.session
|
||||||
|
|
||||||
session.execute("INSERT INTO aircraft_beacon(address, location, altitude, timestamp, track, ground_speed, climb_rate, turn_rate) VALUES('DDEFF7','0101000020E61000009668B61829F12640330E0887F1E94740',604,'2016-07-02 10:47:12',0,0,0,0)")
|
session.execute("INSERT INTO aircraft_beacon(address, location, altitude, timestamp, track, ground_speed, climb_rate, turn_rate) VALUES('DDEFF7','0101000020E61000009668B61829F12640330E0887F1E94740',604,'2016-07-02 10:47:12',0,0,0,0)")
|
||||||
|
@ -95,7 +97,12 @@ class TestDB(unittest.TestCase):
|
||||||
session.execute("UPDATE aircraft_beacon SET device_id = d.id FROM device d WHERE d.address='DDEFF7'")
|
session.execute("UPDATE aircraft_beacon SET device_id = d.id FROM device d WHERE d.address='DDEFF7'")
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
compute_takeoff_and_landing(session)
|
# find the takeoff and the landing
|
||||||
|
update_takeoff_landing(session)
|
||||||
|
self.assertEqual(self.count_takeoff_and_landings(), 2)
|
||||||
|
|
||||||
|
# we should not find the takeoff and the landing again
|
||||||
|
update_takeoff_landing(session)
|
||||||
self.assertEqual(self.count_takeoff_and_landings(), 2)
|
self.assertEqual(self.count_takeoff_and_landings(), 2)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,23 +0,0 @@
|
||||||
import unittest
|
|
||||||
import unittest.mock as mock
|
|
||||||
|
|
||||||
from ogn.gateway.manage import run
|
|
||||||
# from ogn.gateway.manage import import_logfile
|
|
||||||
|
|
||||||
|
|
||||||
class GatewayManagerTest(unittest.TestCase):
|
|
||||||
# try simple user interrupt
|
|
||||||
@mock.patch('ogn.gateway.manage.AprsClient')
|
|
||||||
def test_run_user_interruption(self, mock_aprs_client):
|
|
||||||
instance = mock_aprs_client.return_value
|
|
||||||
instance.run.side_effect = KeyboardInterrupt()
|
|
||||||
|
|
||||||
run(aprs_user="testuser")
|
|
||||||
|
|
||||||
instance.connect.assert_called_once_with()
|
|
||||||
self.assertEqual(instance.run.call_count, 1)
|
|
||||||
instance.disconnect.assert_called_once_with()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
Ładowanie…
Reference in New Issue