From a77cdb6787be6fb53d16da84164237bffc1eb821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 3 Oct 2017 20:20:32 +0200 Subject: [PATCH 01/32] Fix postgis problem --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index dc0fcc3..cb5c26b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ services: before_script: - flake8 tests ogn + - sudo apt-get autoremove postgis* + - sudo apt-get autoremove postgresql* + - sudo apt-get install postgresql-9.3-postgis-2.1 - psql -c 'CREATE DATABASE ogn_test;' -U postgres - psql -c 'CREATE EXTENSION postgis;' -U postgres -d ogn_test From 728beeb3340e714237f1e324e17516f9ea0c1691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 3 Oct 2017 20:32:39 +0200 Subject: [PATCH 02/32] Fix travis --- .travis.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb5c26b..f18c538 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,15 +5,17 @@ env: python: - 3.4 + - 3.5 + - 3.6 -services: - - postgresql +addons: + postgresql: "9.5" + apt: + packages: + - postgresql-9.5-postgis-2.3 before_script: - - flake8 tests ogn - - sudo apt-get autoremove postgis* - - sudo apt-get autoremove postgresql* - - sudo apt-get install postgresql-9.3-postgis-2.1 + - flake8 tests ogn_test - psql -c 'CREATE DATABASE ogn_test;' -U postgres - psql -c 'CREATE EXTENSION postgis;' -U postgres -d ogn_test From ce76a6bb2e03bf570857d0727f2046b7184345bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 6 Oct 2017 21:38:08 +0200 Subject: [PATCH 03/32] Faster bulk import --- ogn/commands/bulkimport.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/ogn/commands/bulkimport.py b/ogn/commands/bulkimport.py index 9d5c2ae..d77a3b1 100644 --- a/ogn/commands/bulkimport.py +++ b/ogn/commands/bulkimport.py @@ -1,4 +1,5 @@ import os +import re from manager import Manager from ogn.commands.dbutils import session @@ -9,18 +10,19 @@ from ogn.utils import open_file manager = Manager() +PATTERN = '^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$' + @manager.command def convert_logfile(path, logfile='main.log', loglevel='INFO'): """Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) . Logfile name: blablabla.txt_YYYY-MM-DD.""" if os.path.isfile(path): - print("Reading file: {}".format(path)) - convert(path) + head, tail = os.path.split(path) + convert(tail, path=head) print("Finished") elif os.path.isdir(path): for filename in os.listdir(path): - print("Reading file: {}".format(filename)) convert(filename, path=path) print("Finished") else: @@ -28,15 +30,25 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'): def convert(sourcefile, path=''): - import re import csv import gzip import datetime - match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile) + match = re.search(PATTERN, sourcefile) if match: reference_date_string = match.group(1) reference_date = datetime.datetime.strptime(reference_date_string, "%Y-%m-%d") + + aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz') + receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz') + + if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename): + print("Reading file: {}".format(sourcefile)) + fout_ab = gzip.open(aircraft_beacon_filename, 'wt') + fout_rb = gzip.open(receiver_beacon_filename, 'wt') + else: + print("Output files for file {} already exists. Skipping".format(sourcefile)) + return else: print("filename '{}' does not match pattern. Skipping".format(sourcefile)) return @@ -49,16 +61,6 @@ def convert(sourcefile, path=''): total += 1 fin.seek(0) - aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz') - receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz') - - if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename): - fout_ab = gzip.open(aircraft_beacon_filename, 'wt') - fout_rb = gzip.open(receiver_beacon_filename, 'wt') - else: - print("Output files already exists. Skipping") - return - aircraft_beacons = list() receiver_beacons = list() From 846372d0253f6aa2a19edd4087260f5bbd320298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sat, 2 Dec 2017 12:43:34 +0100 Subject: [PATCH 04/32] Updated dependencies --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 167a3cd..91b4037 100644 --- a/setup.py +++ b/setup.py @@ -32,22 +32,22 @@ setup( keywords='gliding ogn', packages=find_packages(exclude=['tests', 'tests.*']), install_requires=[ - 'SQLAlchemy==1.1.10', + 'SQLAlchemy==1.1.15', 'geopy==1.11.0', 'manage.py==0.2.10', 'celery[redis]>=3.1,<3.2', - 'alembic==0.9.2', - 'aerofiles==0.3', + 'alembic==0.9.6', + 'aerofiles==0.4', 'geoalchemy2==0.4.0', 'shapely==1.5.17.post1', 'ogn-client==0.8.0', - 'psycopg2==2.7.1' + 'psycopg2==2.7.3.2' ], extras_require={ 'dev': [ 'nose==1.3.7', - 'coveralls==1.1', - 'flake8==3.3.0' + 'coveralls==1.2', + 'flake8==3.5.0' ] }, zip_safe=False From 791b6720e489353bb5a2b35906dd88f558f26c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sat, 2 Dec 2017 12:44:57 +0100 Subject: [PATCH 05/32] Handle NotImplementedError --- ogn/gateway/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ogn/gateway/process.py b/ogn/gateway/process.py index 57b0168..0c4b797 100644 --- a/ogn/gateway/process.py +++ b/ogn/gateway/process.py @@ -31,6 +31,9 @@ def message_to_beacon(raw_message, reference_date): beacon = ReceiverBeacon(**message) else: print("Whoops: what is this: {}".format(message)) + except NotImplementedError as e: + logger.error('Received message: {}'.format(raw_message)) + logger.error(e) except ParseError as e: logger.error('Received message: {}'.format(raw_message)) logger.error('Drop packet, {}'.format(e.message)) From 9e91a825f5a7d2f0b00e0f0bd734180f6af7c4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sat, 2 Dec 2017 13:08:44 +0100 Subject: [PATCH 06/32] Refactoring --- ogn/collect/database.py | 6 +++--- ogn/commands/bulkimport.py | 2 +- ogn/commands/database.py | 6 +++--- ogn/commands/showdeviceinfos.py | 4 ++-- ogn/model/__init__.py | 2 +- ogn/model/{address_origin.py => device_info_origin.py} | 2 +- ogn/utils.py | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) rename ogn/model/{address_origin.py => device_info_origin.py} (96%) diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 1a36dee..0123253 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -1,6 +1,6 @@ from celery.utils.log import get_task_logger -from ogn.model import DeviceInfo, AddressOrigin +from ogn.model import DeviceInfo, DeviceInfoOrigin from ogn.utils import get_ddb from ogn.collect.celery import app @@ -27,7 +27,7 @@ def import_ddb(): """Import registered devices from the DDB.""" logger.info("Import registered devices fom the DDB...") - address_origin = AddressOrigin.ogn_ddb + address_origin = DeviceInfoOrigin.ogn_ddb counter = update_device_infos(app.session, address_origin) logger.info("Imported {} devices.".format(counter)) @@ -38,7 +38,7 @@ def import_file(path='tests/custom_ddb.txt'): """Import registered devices from a local file.""" logger.info("Import registered devices from '{}'...".format(path)) - address_origin = AddressOrigin.user_defined + address_origin = DeviceInfoOrigin.user_defined counter = update_device_infos(app.session, address_origin, csvfile=path) logger.info("Imported {} devices.".format(counter)) diff --git a/ogn/commands/bulkimport.py b/ogn/commands/bulkimport.py index d77a3b1..0a0d3a5 100644 --- a/ogn/commands/bulkimport.py +++ b/ogn/commands/bulkimport.py @@ -184,7 +184,7 @@ def import_logfile(path): else: print("For {} beacons already exist. Skipping".format(reference_date_string)) else: - print("Unknown file type: {}".format()) + print("Unknown file type: {}".format(tail)) def check_no_beacons(tablename, reference_date_string): diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 9af5717..2dfc60b 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -1,7 +1,7 @@ from manager import Manager from ogn.collect.database import update_device_infos from ogn.commands.dbutils import engine, session -from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver from ogn.utils import get_airports from sqlalchemy import insert, distinct from sqlalchemy.sql import null @@ -53,7 +53,7 @@ def import_ddb(): """Import registered devices from the DDB.""" print("Import registered devices fom the DDB...") - address_origin = AddressOrigin.ogn_ddb + address_origin = DeviceInfoOrigin.ogn_ddb counter = update_device_infos(session, address_origin) print("Imported %i devices." % counter) @@ -65,7 +65,7 @@ def import_file(path='tests/custom_ddb.txt'): # (flushes previously manually imported entries) print("Import registered devices from '{}'...".format(path)) - address_origin = AddressOrigin.user_defined + address_origin = DeviceInfoOrigin.user_defined counter = update_device_infos(session, address_origin, csvfile=path) diff --git a/ogn/commands/showdeviceinfos.py b/ogn/commands/showdeviceinfos.py index b39b856..b1a46aa 100644 --- a/ogn/commands/showdeviceinfos.py +++ b/ogn/commands/showdeviceinfos.py @@ -1,6 +1,6 @@ from manager import Manager from ogn.commands.dbutils import session -from ogn.model import AddressOrigin +from ogn.model import DeviceInfoOrigin from ogn.model.device_info import DeviceInfo from sqlalchemy import func, and_, true, false @@ -39,7 +39,7 @@ def get_devices_stats(session): stats = {} for [address_origin, device_count, default_count, nt_count, ni_count, ntni_count] in query.all(): - origin = AddressOrigin(address_origin).name() + origin = DeviceInfoOrigin(address_origin).name() stats[origin] = {'device_count': device_count, 'default_count': default_count, 'nt_count': nt_count, diff --git a/ogn/model/__init__.py b/ogn/model/__init__.py index e83b673..42f8385 100644 --- a/ogn/model/__init__.py +++ b/ogn/model/__init__.py @@ -1,10 +1,10 @@ # flake8: noqa -from .address_origin import AddressOrigin from .aircraft_type import AircraftType from .base import Base from .beacon import Beacon from .device import Device from .device_info import DeviceInfo +from .device_info_origin import DeviceInfoOrigin from .aircraft_beacon import AircraftBeacon from .receiver_beacon import ReceiverBeacon from .receiver import Receiver diff --git a/ogn/model/address_origin.py b/ogn/model/device_info_origin.py similarity index 96% rename from ogn/model/address_origin.py rename to ogn/model/device_info_origin.py index a269973..4105169 100644 --- a/ogn/model/address_origin.py +++ b/ogn/model/device_info_origin.py @@ -1,4 +1,4 @@ -class AddressOrigin: +class DeviceInfoOrigin: unknown = 0 ogn_ddb = 1 flarmnet = 2 diff --git a/ogn/utils.py b/ogn/utils.py index fa7caf2..d5ab334 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -8,7 +8,7 @@ from geopy.geocoders import Nominatim from ogn.parser.utils import feet2m import requests -from .model import AddressOrigin, DeviceInfo, Airport, Location +from .model import DeviceInfoOrigin, DeviceInfo, Airport, Location DDB_URL = "http://ddb.glidernet.org/download/?t=1" @@ -22,7 +22,7 @@ nm2m = 1852 mi2m = 1609.34 -def get_ddb(csvfile=None, address_origin=AddressOrigin.unknown): +def get_ddb(csvfile=None, device_info_origin=DeviceInfoOrigin.unknown): if csvfile is None: r = requests.get(DDB_URL) rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#') @@ -43,7 +43,7 @@ def get_ddb(csvfile=None, address_origin=AddressOrigin.unknown): device_info.tracked = row[5] == 'Y' device_info.identified = row[6] == 'Y' device_info.aircraft_type = int(row[7]) - device_info.address_origin = address_origin + device_info.address_origin = device_info_origin device_infos.append(device_info) From 316522d81c4944974d8a089732be1ed6ee99782e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sat, 2 Dec 2017 16:45:09 +0100 Subject: [PATCH 07/32] Update receivers --- ogn/commands/database.py | 76 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 2dfc60b..eb38a37 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -4,7 +4,7 @@ from ogn.commands.dbutils import engine, session from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver from ogn.utils import get_airports from sqlalchemy import insert, distinct -from sqlalchemy.sql import null +from sqlalchemy.sql import null, and_, or_, func, not_ manager = Manager() @@ -131,3 +131,77 @@ def update_relations(): session.commit() print("Updated {} AircraftBeacons and {} ReceiverBeacons". format(upd, upd2)) + + +@manager.command +def update_receivers(): + """Add/update entries in receiver table and update foreign keys in aircraft beacons and receiver beacons.""" + # Create missing Receiver from ReceiverBeacon + available_receivers = session.query(Receiver.name) \ + .subquery() + + missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .filter(~ReceiverBeacon.name.in_(available_receivers)) + + ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query) + res = session.execute(ins) + insert_count = res.rowcount + + # Update missing or changed location, update it and set country code to None + last_beacon_update = session.query(ReceiverBeacon.name, func.max(ReceiverBeacon.timestamp).label("timestamp")) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .group_by(ReceiverBeacon.name) \ + .subquery() + + last_position = session.query(ReceiverBeacon) \ + .filter(and_(ReceiverBeacon.name == last_beacon_update.c.name, ReceiverBeacon.timestamp == last_beacon_update.c.timestamp, + ReceiverBeacon.location_wkt != null(), ReceiverBeacon.altitude != null())) \ + .subquery() + + location_changed = session.query(last_position) \ + .filter(and_(last_position.c.name == Receiver.name)) \ + .filter(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_position.columns.location)), last_position.c.altitude != Receiver.altitude)) \ + .subquery() + + upd = session.query(Receiver) \ + .filter(Receiver.name == location_changed.c.name) \ + .update({Receiver.location_wkt: location_changed.c.location, + Receiver.altitude: location_changed.c.altitude, + Receiver.country_code: None}, + synchronize_session='fetch') + + # Update missing or changed status + last_status = session.query(ReceiverBeacon) \ + .filter(and_(ReceiverBeacon.name == last_beacon_update.columns.name, ReceiverBeacon.timestamp == last_beacon_update.c.timestamp, + ReceiverBeacon.version != null(), ReceiverBeacon.platform != null())) \ + .subquery() + + status_changed = session.query(last_status) \ + .filter(and_(last_status.columns.name == Receiver.name)) \ + .filter(or_(Receiver.version == null(), Receiver.platform == null(), Receiver.version != last_status.columns.version)) \ + .subquery() + + upd2 = session.query(Receiver) \ + .filter(Receiver.name == status_changed.columns.name) \ + .update({Receiver.version: status_changed.c.version, + Receiver.platform: status_changed.c.platform}, + synchronize_session='fetch') + # Update relations to aircraft beacons + upd3 = session.query(AircraftBeacon) \ + .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ + .update({AircraftBeacon.receiver_id: Receiver.id}, + synchronize_session='fetch') + + # Update relations to receiver beacons + upd4 = session.query(ReceiverBeacon) \ + .filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \ + .update({ReceiverBeacon.receiver_id: Receiver.id}, + synchronize_session='fetch') + + session.commit() + + print("Inserted {} Receivers".format(insert_count)) + print("Updated Receivers: {} positions, {} status".format(upd, upd2)) + print("Updated Relations: {} aircraft beacons, {} receiver beacons".format(upd3, upd4)) + return From 5233144df36725cc616b05669030440ba5b37a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sat, 2 Dec 2017 22:49:12 +0100 Subject: [PATCH 08/32] Added function update_devices --- ogn/commands/database.py | 47 +++++++++++----------------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index eb38a37..b645fef 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -84,53 +84,31 @@ def import_airports(path='tests/SeeYou.cup'): @manager.command -def update_relations(): - """Update AircraftBeacon and ReceiverBeacon relations""" - - # Create missing Receiver from ReceiverBeacon - available_receivers = session.query(Receiver.name) \ - .subquery() - - missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .filter(~ReceiverBeacon.name.in_(available_receivers)) - - ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query) - session.execute(ins) +def update_devices(): + """Add/update entries in devices table and update foreign keys in aircraft beacons.""" # Create missing Device from AircraftBeacon - available_addresses = session.query(Device.address) \ + available_devices = session.query(Device.address) \ .subquery() - missing_addresses_query = session.query(distinct(AircraftBeacon.address)) \ + missing_devices_query = session.query(distinct(AircraftBeacon.address)) \ .filter(AircraftBeacon.device_id == null()) \ - .filter(~AircraftBeacon.address.in_(available_addresses)) + .filter(~AircraftBeacon.address.in_(available_devices)) - ins2 = insert(Device).from_select([Device.address], missing_addresses_query) - session.execute(ins2) - session.commit() - print("Inserted {} Receivers and {} Devices".format(ins, ins2)) - return + ins = insert(Device).from_select([Device.address], missing_devices_query) + res = session.execute(ins) + insert_count = res.rowcount - # Update AircraftBeacons + # Update relations to aircraft beacons upd = session.query(AircraftBeacon) \ .filter(AircraftBeacon.device_id == null()) \ - .filter(AircraftBeacon.receiver_id == null()) \ .filter(AircraftBeacon.address == Device.address) \ - .filter(AircraftBeacon.receiver_name == Receiver.name) \ - .update({AircraftBeacon.device_id: Device.id, - AircraftBeacon.receiver_id: Receiver.id}, - synchronize_session='fetch') - - upd2 = session.query(ReceiverBeacon) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .filter(ReceiverBeacon.receiver_name == Receiver.name) \ - .update({Receiver.name: ReceiverBeacon.receiver_name}, + .update({AircraftBeacon.device_id: Device.id}, synchronize_session='fetch') session.commit() - print("Updated {} AircraftBeacons and {} ReceiverBeacons". - format(upd, upd2)) + print("Inserted {} Devices".format(insert_count)) + print("Updated {} AircraftBeacons".format(upd)) @manager.command @@ -187,6 +165,7 @@ def update_receivers(): .update({Receiver.version: status_changed.c.version, Receiver.platform: status_changed.c.platform}, synchronize_session='fetch') + # Update relations to aircraft beacons upd3 = session.query(AircraftBeacon) \ .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ From bc2c70d34551b29dcff4a77d98ec7e2774fe23c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sun, 3 Dec 2017 20:05:06 +0100 Subject: [PATCH 09/32] Merged status and position update, added update country code command --- ogn/commands/database.py | 119 +++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index b645fef..7ccc05b 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -5,6 +5,9 @@ from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, De from ogn.utils import get_airports from sqlalchemy import insert, distinct from sqlalchemy.sql import null, and_, or_, func, not_ +from sqlalchemy.sql.expression import case + +from ogn.utils import get_country_code manager = Manager() @@ -113,7 +116,7 @@ def update_devices(): @manager.command def update_receivers(): - """Add/update entries in receiver table and update foreign keys in aircraft beacons and receiver beacons.""" + """Add/update_receivers entries in receiver table and update_receivers foreign keys in aircraft beacons and receiver beacons.""" # Create missing Receiver from ReceiverBeacon available_receivers = session.query(Receiver.name) \ .subquery() @@ -126,61 +129,95 @@ def update_receivers(): res = session.execute(ins) insert_count = res.rowcount - # Update missing or changed location, update it and set country code to None - last_beacon_update = session.query(ReceiverBeacon.name, func.max(ReceiverBeacon.timestamp).label("timestamp")) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .group_by(ReceiverBeacon.name) \ - .subquery() + # Update missing or changed values, update_receivers them and set country code to None if location changed + new_values_range = session.query(ReceiverBeacon.name, + func.min(ReceiverBeacon.timestamp).label('firstseen'), + func.max(ReceiverBeacon.timestamp).label('lastseen')) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .group_by(ReceiverBeacon.name) \ + .subquery() - last_position = session.query(ReceiverBeacon) \ - .filter(and_(ReceiverBeacon.name == last_beacon_update.c.name, ReceiverBeacon.timestamp == last_beacon_update.c.timestamp, - ReceiverBeacon.location_wkt != null(), ReceiverBeacon.altitude != null())) \ - .subquery() + last_values = session.query(ReceiverBeacon.name, + func.max(new_values_range.c.firstseen).label('firstseen'), + func.max(new_values_range.c.lastseen).label('lastseen'), + func.max(ReceiverBeacon.location_wkt).label('location_wkt'), + func.max(ReceiverBeacon.altitude).label('altitude'), + func.max(ReceiverBeacon.version).label('version'), + func.max(ReceiverBeacon.platform).label('platform')) \ + .filter(and_(ReceiverBeacon.name == new_values_range.c.name, + ReceiverBeacon.timestamp == new_values_range.c.lastseen)) \ + .group_by(ReceiverBeacon.name) \ + .subquery() - location_changed = session.query(last_position) \ - .filter(and_(last_position.c.name == Receiver.name)) \ - .filter(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_position.columns.location)), last_position.c.altitude != Receiver.altitude)) \ - .subquery() + last_valid_values = session.query(last_values) \ + .filter(and_(last_values.c.firstseen != null(), + last_values.c.lastseen != null(), + last_values.c.location_wkt != null(), + last_values.c.altitude != null(), + last_values.c.version != null(), + last_values.c.platform != null())) \ + .subquery() - upd = session.query(Receiver) \ - .filter(Receiver.name == location_changed.c.name) \ - .update({Receiver.location_wkt: location_changed.c.location, - Receiver.altitude: location_changed.c.altitude, - Receiver.country_code: None}, - synchronize_session='fetch') + update_values = session.query(Receiver.name, + case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), + (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), + case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), + (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ + .filter(Receiver.name == last_valid_values.c.name) \ + .subquery() - # Update missing or changed status - last_status = session.query(ReceiverBeacon) \ - .filter(and_(ReceiverBeacon.name == last_beacon_update.columns.name, ReceiverBeacon.timestamp == last_beacon_update.c.timestamp, - ReceiverBeacon.version != null(), ReceiverBeacon.platform != null())) \ - .subquery() - - status_changed = session.query(last_status) \ - .filter(and_(last_status.columns.name == Receiver.name)) \ - .filter(or_(Receiver.version == null(), Receiver.platform == null(), Receiver.version != last_status.columns.version)) \ - .subquery() - - upd2 = session.query(Receiver) \ - .filter(Receiver.name == status_changed.columns.name) \ - .update({Receiver.version: status_changed.c.version, - Receiver.platform: status_changed.c.platform}, + update_receivers = session.query(Receiver) \ + .filter(Receiver.name == update_values.c.name) \ + .update({Receiver.firstseen: update_values.c.firstseen, + Receiver.lastseen: update_values.c.lastseen, + Receiver.location_wkt: update_values.c.location_wkt, + Receiver.altitude: update_values.c.altitude, + Receiver.version: update_values.c.version, + Receiver.platform: update_values.c.platform, + Receiver.country_code: update_values.c.country_code}, synchronize_session='fetch') # Update relations to aircraft beacons - upd3 = session.query(AircraftBeacon) \ + update_aircraft_beacons = session.query(AircraftBeacon) \ .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ .update({AircraftBeacon.receiver_id: Receiver.id}, synchronize_session='fetch') # Update relations to receiver beacons - upd4 = session.query(ReceiverBeacon) \ + update_receiver_beacons = session.query(ReceiverBeacon) \ .filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \ .update({ReceiverBeacon.receiver_id: Receiver.id}, synchronize_session='fetch') session.commit() - print("Inserted {} Receivers".format(insert_count)) - print("Updated Receivers: {} positions, {} status".format(upd, upd2)) - print("Updated Relations: {} aircraft beacons, {} receiver beacons".format(upd3, upd4)) - return + print("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) + print("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) + + +@manager.command +def update_country_code(): + # update country code if None + unknown_country_query = session.query(Receiver) \ + .filter(Receiver.country_code == null()) \ + .filter(Receiver.location_wkt != null()) \ + .order_by(Receiver.name) + + for receiver in unknown_country_query.all(): + location = receiver.location + country_code = get_country_code(location.latitude, location.longitude) + if country_code is not None: + receiver.country_code = country_code + print("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + + session.commit() From e527531d00f88b9086ce9dc0aec8b327d8acbf79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sun, 3 Dec 2017 20:49:30 +0100 Subject: [PATCH 10/32] Create device only if error_count == 0 --- ogn/commands/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 7ccc05b..f12be0f 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -95,7 +95,7 @@ def update_devices(): .subquery() missing_devices_query = session.query(distinct(AircraftBeacon.address)) \ - .filter(AircraftBeacon.device_id == null()) \ + .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ .filter(~AircraftBeacon.address.in_(available_devices)) ins = insert(Device).from_select([Device.address], missing_devices_query) @@ -129,7 +129,7 @@ def update_receivers(): res = session.execute(ins) insert_count = res.rowcount - # Update missing or changed values, update_receivers them and set country code to None if location changed + # Update missing or changed values, update them and set country code to None if location changed new_values_range = session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ From e46be20df653fd4420b6dec51e556c07028bebe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Mon, 4 Dec 2017 08:30:31 +0100 Subject: [PATCH 11/32] Correct travis --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f18c538..71fa3f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,9 @@ addons: before_script: - flake8 tests ogn_test - - psql -c 'CREATE DATABASE ogn_test;' -U postgres - - psql -c 'CREATE EXTENSION postgis;' -U postgres -d ogn_test + - psql -U postgres -c 'CREATE DATABASE ogn_test;' + - psql -U postgres -c 'CREATE EXTENSION postgis;' + - psql -U postgres -d ogn_test script: - nosetests --with-coverage --cover-package=ogn From 7407d3a8141e9fae6597dc4ae7d5bb65e8c3df1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Mon, 4 Dec 2017 08:54:17 +0100 Subject: [PATCH 12/32] Corrected dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 91b4037..1c6ad32 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( 'alembic==0.9.6', 'aerofiles==0.4', 'geoalchemy2==0.4.0', - 'shapely==1.5.17.post1', + 'shapely>=1.5.17,<1.6', 'ogn-client==0.8.0', 'psycopg2==2.7.3.2' ], From 5dee579378852a635240befc87d8ff065097c462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Mon, 4 Dec 2017 21:14:56 +0100 Subject: [PATCH 13/32] update receivers and devices with last valid values --- ogn/commands/database.py | 110 ++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index f12be0f..971d9c6 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -101,6 +101,62 @@ def update_devices(): ins = insert(Device).from_select([Device.address], missing_devices_query) res = session.execute(ins) insert_count = res.rowcount + session.commit() + + # For each address in the new beacons: get firstseen, lastseen and last values != NULL + last_valid_values = session.query(distinct(AircraftBeacon.address).label('address'), + func.first_value(AircraftBeacon.timestamp) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) \ + .label('firstseen'), + func.last_value(AircraftBeacon.timestamp) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) \ + .label('lastseen'), + func.first_value(AircraftBeacon.aircraft_type) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None), (AircraftBeacon.aircraft_type != null(), AircraftBeacon.aircraft_type)])) \ + .label('aircraft_type'), + func.first_value(AircraftBeacon.stealth) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None), (AircraftBeacon.stealth != null(), AircraftBeacon.stealth)])) \ + .label('stealth'), + func.first_value(AircraftBeacon.software_version) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None), (AircraftBeacon.software_version != null(), AircraftBeacon.software_version)])) \ + .label('software_version'), + func.first_value(AircraftBeacon.hardware_version) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None), (AircraftBeacon.hardware_version != null(), AircraftBeacon.hardware_version)])) \ + .label('hardware_version'), + func.first_value(AircraftBeacon.real_address) \ + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)])) \ + .label('real_address')) \ + .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ + .subquery() + + update_values = session.query(Device.address, + case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), + case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), + case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), + (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), + case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), + (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), + case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), + case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), + case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), + (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ + .filter(Device.address == last_valid_values.c.address) \ + .subquery() + + update_receivers = session.query(Device) \ + .filter(Device.address == update_values.c.address) \ + .update({Device.firstseen: update_values.c.firstseen, + Device.lastseen: update_values.c.lastseen, + Device.aircraft_type: update_values.c.aircraft_type, + Device.stealth: update_values.c.stealth, + Device.software_version: update_values.c.software_version, + Device.hardware_version: update_values.c.hardware_version, + Device.real_address: update_values.c.real_address}, + synchronize_session='fetch') # Update relations to aircraft beacons upd = session.query(AircraftBeacon) \ @@ -110,7 +166,7 @@ def update_devices(): synchronize_session='fetch') session.commit() - print("Inserted {} Devices".format(insert_count)) + print("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) print("Updated {} AircraftBeacons".format(upd)) @@ -129,34 +185,28 @@ def update_receivers(): res = session.execute(ins) insert_count = res.rowcount - # Update missing or changed values, update them and set country code to None if location changed - new_values_range = session.query(ReceiverBeacon.name, - func.min(ReceiverBeacon.timestamp).label('firstseen'), - func.max(ReceiverBeacon.timestamp).label('lastseen')) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .group_by(ReceiverBeacon.name) \ - .subquery() - - last_values = session.query(ReceiverBeacon.name, - func.max(new_values_range.c.firstseen).label('firstseen'), - func.max(new_values_range.c.lastseen).label('lastseen'), - func.max(ReceiverBeacon.location_wkt).label('location_wkt'), - func.max(ReceiverBeacon.altitude).label('altitude'), - func.max(ReceiverBeacon.version).label('version'), - func.max(ReceiverBeacon.platform).label('platform')) \ - .filter(and_(ReceiverBeacon.name == new_values_range.c.name, - ReceiverBeacon.timestamp == new_values_range.c.lastseen)) \ - .group_by(ReceiverBeacon.name) \ - .subquery() - - last_valid_values = session.query(last_values) \ - .filter(and_(last_values.c.firstseen != null(), - last_values.c.lastseen != null(), - last_values.c.location_wkt != null(), - last_values.c.altitude != null(), - last_values.c.version != null(), - last_values.c.platform != null())) \ - .subquery() + # For each name in the new beacons: get firstseen, lastseen and last values != NULL + last_valid_values = session.query(distinct(ReceiverBeacon.name).label('name'), + func.first_value(ReceiverBeacon.timestamp) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) \ + .label('firstseen'), + func.last_value(ReceiverBeacon.timestamp) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) \ + .label('lastseen'), + func.first_value(ReceiverBeacon.location_wkt) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None), (ReceiverBeacon.location_wkt != null(), ReceiverBeacon.location_wkt)])) \ + .label('location_wkt'), + func.first_value(ReceiverBeacon.altitude) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None), (ReceiverBeacon.altitude != null(), ReceiverBeacon.altitude)])) \ + .label('altitude'), + func.first_value(ReceiverBeacon.version) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None), (ReceiverBeacon.version != null(), ReceiverBeacon.version)])) \ + .label('version'), + func.first_value(ReceiverBeacon.platform) \ + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None), (ReceiverBeacon.platform != null(), ReceiverBeacon.platform)])) \ + .label('platform')) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .subquery() update_values = session.query(Receiver.name, case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), @@ -171,7 +221,7 @@ def update_receivers(): (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), - case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), + case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ .filter(Receiver.name == last_valid_values.c.name) \ .subquery() From ca1dba3e232bc55bf73ab147dc9a285471cf892d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 5 Dec 2017 08:53:38 +0100 Subject: [PATCH 14/32] Added firstseen/lastseen to device --- ogn/model/device.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ogn/model/device.py b/ogn/model/device.py index 491817c..8bbe02b 100644 --- a/ogn/model/device.py +++ b/ogn/model/device.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger +from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger, DateTime from sqlalchemy.orm import relationship from .base import Base @@ -9,6 +9,8 @@ class Device(Base): id = Column(Integer, primary_key=True) address = Column(String(6), index=True) + firstseen = Column(DateTime, index=True) + lastseen = Column(DateTime, index=True) aircraft_type = Column(SmallInteger, index=True) stealth = Column(Boolean) software_version = Column(Float) From 061c3e253e62e6746962319d247562e5bfa2a0c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 08:25:03 +0100 Subject: [PATCH 15/32] Calculate distance between aircraft_beacon and receiver --- ogn/commands/database.py | 5 +++-- ogn/model/aircraft_beacon.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 971d9c6..9aa246a 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -172,7 +172,7 @@ def update_devices(): @manager.command def update_receivers(): - """Add/update_receivers entries in receiver table and update_receivers foreign keys in aircraft beacons and receiver beacons.""" + """Add/update_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons.""" # Create missing Receiver from ReceiverBeacon available_receivers = session.query(Receiver.name) \ .subquery() @@ -240,7 +240,8 @@ def update_receivers(): # Update relations to aircraft beacons update_aircraft_beacons = session.query(AircraftBeacon) \ .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ - .update({AircraftBeacon.receiver_id: Receiver.id}, + .update({AircraftBeacon.receiver_id: Receiver.id, + AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)}, synchronize_session='fetch') # Update relations to receiver beacons diff --git a/ogn/model/aircraft_beacon.py b/ogn/model/aircraft_beacon.py index f0eea61..f75360e 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn/model/aircraft_beacon.py @@ -38,6 +38,9 @@ class AircraftBeacon(Beacon): status = Column(SmallInteger, index=True) + # Calculated values + distance = Column(Float) + # Relations receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True) receiver = relationship('Receiver', foreign_keys=[receiver_id]) From c00812c141f457ba0347ed6afd426f9780dea341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 19:24:33 +0100 Subject: [PATCH 16/32] Refactoring --- ogn/collect/celery.py | 1 - ogn/collect/database.py | 197 ++++++++++++++++++++++++++++++++++++- ogn/commands/database.py | 206 ++++----------------------------------- ogn/utils.py | 4 +- 4 files changed, 217 insertions(+), 191 deletions(-) diff --git a/ogn/collect/celery.py b/ogn/collect/celery.py index c49509c..fca88a1 100644 --- a/ogn/collect/celery.py +++ b/ogn/collect/celery.py @@ -28,7 +28,6 @@ app = Celery('ogn.collect', include=["ogn.collect.database", "ogn.collect.logbook", "ogn.collect.takeoff_landing", - "ogn.collect.receiver" ]) app.config_from_envvar("OGN_CONFIG_MODULE") diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 0123253..7a99968 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -1,10 +1,13 @@ from celery.utils.log import get_task_logger -from ogn.model import DeviceInfo, DeviceInfoOrigin -from ogn.utils import get_ddb +from sqlalchemy import insert, distinct +from sqlalchemy.sql import null, and_, or_, func, not_ +from sqlalchemy.sql.expression import case -from ogn.collect.celery import app +from ogn.model import DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn.utils import get_ddb, get_country_code +from .celery import app logger = get_task_logger(__name__) @@ -42,3 +45,191 @@ def import_file(path='tests/custom_ddb.txt'): counter = update_device_infos(app.session, address_origin, csvfile=path) logger.info("Imported {} devices.".format(counter)) + + +@app.task +def update_devices(): + """Add/update entries in devices table and update foreign keys in aircraft beacons.""" + + # Create missing Device from AircraftBeacon + available_devices = app.session.query(Device.address) \ + .subquery() + + missing_devices_query = app.session.query(distinct(AircraftBeacon.address)) \ + .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ + .filter(~AircraftBeacon.address.in_(available_devices)) + + ins = insert(Device).from_select([Device.address], missing_devices_query) + res = app.session.execute(ins) + insert_count = res.rowcount + app.session.commit() + + # For each address in the new beacons: get firstseen, lastseen and last values != NULL + last_valid_values = app.session.query(distinct(AircraftBeacon.address).label('address'), + func.first_value(AircraftBeacon.timestamp) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) + .label('firstseen'), + func.last_value(AircraftBeacon.timestamp) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) + .label('lastseen'), + func.first_value(AircraftBeacon.aircraft_type) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None), (AircraftBeacon.aircraft_type != null(), AircraftBeacon.aircraft_type)])) + .label('aircraft_type'), + func.first_value(AircraftBeacon.stealth) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None), (AircraftBeacon.stealth != null(), AircraftBeacon.stealth)])) + .label('stealth'), + func.first_value(AircraftBeacon.software_version) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None), (AircraftBeacon.software_version != null(), AircraftBeacon.software_version)])) + .label('software_version'), + func.first_value(AircraftBeacon.hardware_version) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None), (AircraftBeacon.hardware_version != null(), AircraftBeacon.hardware_version)])) + .label('hardware_version'), + func.first_value(AircraftBeacon.real_address) + .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)])) + .label('real_address')) \ + .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ + .subquery() + + update_values = app.session.query(Device.address, + case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), + case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), + case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), + (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), + case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), + (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), + case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), + case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), + case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), + (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ + .filter(Device.address == last_valid_values.c.address) \ + .subquery() + + update_receivers = app.session.query(Device) \ + .filter(Device.address == update_values.c.address) \ + .update({Device.firstseen: update_values.c.firstseen, + Device.lastseen: update_values.c.lastseen, + Device.aircraft_type: update_values.c.aircraft_type, + Device.stealth: update_values.c.stealth, + Device.software_version: update_values.c.software_version, + Device.hardware_version: update_values.c.hardware_version, + Device.real_address: update_values.c.real_address}, + synchronize_session='fetch') + + # Update relations to aircraft beacons + upd = app.session.query(AircraftBeacon) \ + .filter(AircraftBeacon.device_id == null()) \ + .filter(AircraftBeacon.address == Device.address) \ + .update({AircraftBeacon.device_id: Device.id}, + synchronize_session='fetch') + + app.session.commit() + print("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) + print("Updated {} AircraftBeacons".format(upd)) + + +@app.task +def update_receivers(): + """Add/update_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons.""" + # Create missing Receiver from ReceiverBeacon + available_receivers = app.session.query(Receiver.name) \ + .subquery() + + missing_receiver_query = app.session.query(distinct(ReceiverBeacon.name)) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .filter(~ReceiverBeacon.name.in_(available_receivers)) + + ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query) + res = app.session.execute(ins) + insert_count = res.rowcount + + # For each name in the new beacons: get firstseen, lastseen and last values != NULL + last_valid_values = app.session.query(distinct(ReceiverBeacon.name).label('name'), + func.first_value(ReceiverBeacon.timestamp) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) + .label('firstseen'), + func.last_value(ReceiverBeacon.timestamp) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) + .label('lastseen'), + func.first_value(ReceiverBeacon.location_wkt) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None), (ReceiverBeacon.location_wkt != null(), ReceiverBeacon.location_wkt)])) + .label('location_wkt'), + func.first_value(ReceiverBeacon.altitude) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None), (ReceiverBeacon.altitude != null(), ReceiverBeacon.altitude)])) + .label('altitude'), + func.first_value(ReceiverBeacon.version) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None), (ReceiverBeacon.version != null(), ReceiverBeacon.version)])) + .label('version'), + func.first_value(ReceiverBeacon.platform) + .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None), (ReceiverBeacon.platform != null(), ReceiverBeacon.platform)])) + .label('platform')) \ + .filter(ReceiverBeacon.receiver_id == null()) \ + .subquery() + + update_values = app.session.query(Receiver.name, + case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), + (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), + case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed + (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ + .filter(Receiver.name == last_valid_values.c.name) \ + .subquery() + + update_receivers = app.session.query(Receiver) \ + .filter(Receiver.name == update_values.c.name) \ + .update({Receiver.firstseen: update_values.c.firstseen, + Receiver.lastseen: update_values.c.lastseen, + Receiver.location_wkt: update_values.c.location_wkt, + Receiver.altitude: update_values.c.altitude, + Receiver.version: update_values.c.version, + Receiver.platform: update_values.c.platform, + Receiver.country_code: update_values.c.country_code}, + synchronize_session='fetch') + + # Update relations to aircraft beacons + update_aircraft_beacons = app.session.query(AircraftBeacon) \ + .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ + .update({AircraftBeacon.receiver_id: Receiver.id, + AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)}, + synchronize_session='fetch') + + # Update relations to receiver beacons + update_receiver_beacons = app.session.query(ReceiverBeacon) \ + .filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \ + .update({ReceiverBeacon.receiver_id: Receiver.id}, + synchronize_session='fetch') + + app.session.commit() + + print("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) + print("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) + + +@app.task +def update_country_code(): + # update country code if None + unknown_country_query = app.session.query(Receiver) \ + .filter(Receiver.country_code == null()) \ + .filter(Receiver.location_wkt != null()) \ + .order_by(Receiver.name) + + for receiver in unknown_country_query.all(): + location = receiver.location + country_code = get_country_code(location.latitude, location.longitude) + if country_code is not None: + receiver.country_code = country_code + print("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + + app.session.commit() diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 9aa246a..f01cd98 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -1,13 +1,10 @@ from manager import Manager from ogn.collect.database import update_device_infos from ogn.commands.dbutils import engine, session -from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon from ogn.utils import get_airports -from sqlalchemy import insert, distinct -from sqlalchemy.sql import null, and_, or_, func, not_ -from sqlalchemy.sql.expression import case - -from ogn.utils import get_country_code +from sqlalchemy import distinct +from sqlalchemy.sql import null, func manager = Manager() @@ -86,189 +83,28 @@ def import_airports(path='tests/SeeYou.cup'): print("Imported {} airports.".format(len(airports))) -@manager.command -def update_devices(): - """Add/update entries in devices table and update foreign keys in aircraft beacons.""" - - # Create missing Device from AircraftBeacon - available_devices = session.query(Device.address) \ - .subquery() - - missing_devices_query = session.query(distinct(AircraftBeacon.address)) \ - .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ - .filter(~AircraftBeacon.address.in_(available_devices)) - - ins = insert(Device).from_select([Device.address], missing_devices_query) - res = session.execute(ins) - insert_count = res.rowcount - session.commit() - - # For each address in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = session.query(distinct(AircraftBeacon.address).label('address'), - func.first_value(AircraftBeacon.timestamp) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) \ - .label('firstseen'), - func.last_value(AircraftBeacon.timestamp) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) \ - .label('lastseen'), - func.first_value(AircraftBeacon.aircraft_type) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.aircraft_type == null(), None), (AircraftBeacon.aircraft_type != null(), AircraftBeacon.aircraft_type)])) \ - .label('aircraft_type'), - func.first_value(AircraftBeacon.stealth) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.stealth == null(), None), (AircraftBeacon.stealth != null(), AircraftBeacon.stealth)])) \ - .label('stealth'), - func.first_value(AircraftBeacon.software_version) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.software_version == null(), None), (AircraftBeacon.software_version != null(), AircraftBeacon.software_version)])) \ - .label('software_version'), - func.first_value(AircraftBeacon.hardware_version) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.hardware_version == null(), None), (AircraftBeacon.hardware_version != null(), AircraftBeacon.hardware_version)])) \ - .label('hardware_version'), - func.first_value(AircraftBeacon.real_address) \ - .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)])) \ - .label('real_address')) \ - .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ - .subquery() - - update_values = session.query(Device.address, - case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), - case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), - case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), - (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), - case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), - (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), - case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), - case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), - case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), - (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ - .filter(Device.address == last_valid_values.c.address) \ - .subquery() - - update_receivers = session.query(Device) \ - .filter(Device.address == update_values.c.address) \ - .update({Device.firstseen: update_values.c.firstseen, - Device.lastseen: update_values.c.lastseen, - Device.aircraft_type: update_values.c.aircraft_type, - Device.stealth: update_values.c.stealth, - Device.software_version: update_values.c.software_version, - Device.hardware_version: update_values.c.hardware_version, - Device.real_address: update_values.c.real_address}, - synchronize_session='fetch') - - # Update relations to aircraft beacons - upd = session.query(AircraftBeacon) \ - .filter(AircraftBeacon.device_id == null()) \ - .filter(AircraftBeacon.address == Device.address) \ - .update({AircraftBeacon.device_id: Device.id}, - synchronize_session='fetch') - - session.commit() - print("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) - print("Updated {} AircraftBeacons".format(upd)) - - @manager.command def update_receivers(): - """Add/update_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons.""" - # Create missing Receiver from ReceiverBeacon - available_receivers = session.query(Receiver.name) \ - .subquery() - - missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .filter(~ReceiverBeacon.name.in_(available_receivers)) - - ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query) - res = session.execute(ins) - insert_count = res.rowcount - - # For each name in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = session.query(distinct(ReceiverBeacon.name).label('name'), - func.first_value(ReceiverBeacon.timestamp) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) \ - .label('firstseen'), - func.last_value(ReceiverBeacon.timestamp) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) \ - .label('lastseen'), - func.first_value(ReceiverBeacon.location_wkt) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.location_wkt == null(), None), (ReceiverBeacon.location_wkt != null(), ReceiverBeacon.location_wkt)])) \ - .label('location_wkt'), - func.first_value(ReceiverBeacon.altitude) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.altitude == null(), None), (ReceiverBeacon.altitude != null(), ReceiverBeacon.altitude)])) \ - .label('altitude'), - func.first_value(ReceiverBeacon.version) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.version == null(), None), (ReceiverBeacon.version != null(), ReceiverBeacon.version)])) \ - .label('version'), - func.first_value(ReceiverBeacon.platform) \ - .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.platform == null(), None), (ReceiverBeacon.platform != null(), ReceiverBeacon.platform)])) \ - .label('platform')) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .subquery() - - update_values = session.query(Receiver.name, - case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), - (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), - case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed - (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ - .filter(Receiver.name == last_valid_values.c.name) \ - .subquery() - - update_receivers = session.query(Receiver) \ - .filter(Receiver.name == update_values.c.name) \ - .update({Receiver.firstseen: update_values.c.firstseen, - Receiver.lastseen: update_values.c.lastseen, - Receiver.location_wkt: update_values.c.location_wkt, - Receiver.altitude: update_values.c.altitude, - Receiver.version: update_values.c.version, - Receiver.platform: update_values.c.platform, - Receiver.country_code: update_values.c.country_code}, - synchronize_session='fetch') - - # Update relations to aircraft beacons - update_aircraft_beacons = session.query(AircraftBeacon) \ - .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ - .update({AircraftBeacon.receiver_id: Receiver.id, - AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)}, - synchronize_session='fetch') - - # Update relations to receiver beacons - update_receiver_beacons = session.query(ReceiverBeacon) \ - .filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \ - .update({ReceiverBeacon.receiver_id: Receiver.id}, - synchronize_session='fetch') - - session.commit() - - print("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) - print("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) + from ogn.collect.database import update_receivers as ur + ur() @manager.command -def update_country_code(): - # update country code if None - unknown_country_query = session.query(Receiver) \ - .filter(Receiver.country_code == null()) \ - .filter(Receiver.location_wkt != null()) \ - .order_by(Receiver.name) +def update_receiver_stats(): + """Add/update entries in receiver stats table.""" - for receiver in unknown_country_query.all(): - location = receiver.location - country_code = get_country_code(location.latitude, location.longitude) - if country_code is not None: - receiver.country_code = country_code - print("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + asdf = session.query(ReceiverBeacon.receiver_id, + func.count(distinct(AircraftBeacon.device_id)).label('device_count'), + func.max(AircraftBeacon.altitude).label('max_altitude'), + func.max(func.ST_Distance(AircraftBeacon.location_wkt, AircraftBeacon.location_wkt)).label('max_distance')) \ + .filter(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id) \ + .group_by(ReceiverBeacon.id) - session.commit() + print(asdf) + for a in asdf.all(): + print(a) + + return + + asdf = session.query(distinct(ReceiverBeacon.receiver_id), func.DATE(ReceiverBeacon.timestamp).label('date')) \ + .filter(ReceiverBeacon.receiver_id != null()) diff --git a/ogn/utils.py b/ogn/utils.py index d5ab334..8cd4cfd 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -22,7 +22,7 @@ nm2m = 1852 mi2m = 1609.34 -def get_ddb(csvfile=None, device_info_origin=DeviceInfoOrigin.unknown): +def get_ddb(csvfile=None, address_origin=DeviceInfoOrigin.unknown): if csvfile is None: r = requests.get(DDB_URL) rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#') @@ -43,7 +43,7 @@ def get_ddb(csvfile=None, device_info_origin=DeviceInfoOrigin.unknown): device_info.tracked = row[5] == 'Y' device_info.identified = row[6] == 'Y' device_info.aircraft_type = int(row[7]) - device_info.address_origin = device_info_origin + device_info.address_origin = address_origin device_infos.append(device_info) From 61d6e7110977b6080c627d1508c046cd6becdd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 19:38:10 +0100 Subject: [PATCH 17/32] whoops... stashed --- ogn/model/beacon.py | 2 +- ogn/model/device.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index d36dbe5..4ed46d3 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -16,7 +16,7 @@ class Beacon(AbstractConcreteBase, Base): name = Column(String) receiver_name = Column(String(9)) - dstcall = None + dstcall = Column(String) timestamp = Column(DateTime, index=True) symboltable = None symbolcode = None diff --git a/ogn/model/device.py b/ogn/model/device.py index 8bbe02b..753e481 100644 --- a/ogn/model/device.py +++ b/ogn/model/device.py @@ -16,6 +16,8 @@ class Device(Base): software_version = Column(Float) hardware_version = Column(SmallInteger) real_address = Column(String(6)) + firstseen = Column(DateTime, index=True) + lastseen = Column(DateTime, index=True) # Relations aircraft_beacons = relationship('AircraftBeacon') From 0b8c0e971c7a79912feebe23282dabbe089c37c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 20:12:00 +0100 Subject: [PATCH 18/32] removed old receiver update code --- ogn/collect/receiver.py | 111 ---------------------------------------- 1 file changed, 111 deletions(-) delete mode 100644 ogn/collect/receiver.py diff --git a/ogn/collect/receiver.py b/ogn/collect/receiver.py deleted file mode 100644 index 81fdf83..0000000 --- a/ogn/collect/receiver.py +++ /dev/null @@ -1,111 +0,0 @@ -from sqlalchemy.sql import func, null -from sqlalchemy.sql.functions import coalesce -from sqlalchemy import and_, not_, or_ - -from celery.utils.log import get_task_logger - -from ogn.model import Receiver, ReceiverBeacon -from ogn.utils import get_country_code -from ogn.collect.celery import app - -logger = get_task_logger(__name__) - - -@app.task -def update_receivers(): - """Update the receiver table.""" - # get the timestamp of last update - last_update_query = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) - last_update = last_update_query.one().last_entry - - # get last receiver beacons since last update - last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, - func.max(ReceiverBeacon.timestamp).label('lastseen')) \ - .filter(ReceiverBeacon.timestamp >= last_update) \ - .group_by(ReceiverBeacon.name) \ - .subquery() - - # update receivers - receivers_to_update = app.session.query(ReceiverBeacon.name, - ReceiverBeacon.location_wkt, - ReceiverBeacon.altitude, - last_receiver_beacon_sq.columns.lastseen, - ReceiverBeacon.version, - ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, - ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \ - .subquery() - - # ... set country code to None if lat or lon changed - changed_count = app.session.query(Receiver) \ - .filter(Receiver.name == receivers_to_update.columns.name) \ - .filter(or_(not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)), - and_(Receiver.location_wkt == null(), - receivers_to_update.columns.location != null()))) \ - .update({"location_wkt": receivers_to_update.columns.location, - "country_code": null()}, - synchronize_session=False) - - # ... and update altitude, lastseen, version and platform - update_count = app.session.query(Receiver) \ - .filter(Receiver.name == receivers_to_update.columns.name) \ - .update({"altitude": receivers_to_update.columns.altitude, - "lastseen": receivers_to_update.columns.lastseen, - "version": receivers_to_update.columns.version, - "platform": receivers_to_update.columns.platform}) - - # add new receivers - empty_sq = app.session.query(ReceiverBeacon.name, - ReceiverBeacon.location_wkt, - ReceiverBeacon.altitude, - last_receiver_beacon_sq.columns.lastseen, - ReceiverBeacon.version, ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, - ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \ - .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \ - .filter(Receiver.name == null()) \ - .order_by(ReceiverBeacon.name) - - for receiver_beacon in empty_sq.all(): - receiver = Receiver() - receiver.name = receiver_beacon.name - receiver.location_wkt = receiver_beacon.location_wkt - receiver.altitude = receiver_beacon.altitude - receiver.firstseen = None - receiver.lastseen = receiver_beacon.lastseen - receiver.version = receiver_beacon.version - receiver.platform = receiver_beacon.platform - - app.session.add(receiver) - logger.info("{} added".format(receiver.name)) - - # update firstseen if None - firstseen_null_query = app.session.query(Receiver.name, - func.min(ReceiverBeacon.timestamp).label('firstseen')) \ - .filter(Receiver.firstseen == null()) \ - .join(ReceiverBeacon, Receiver.name == ReceiverBeacon.name) \ - .group_by(Receiver.name) \ - .subquery() - - added_count = app.session.query(Receiver) \ - .filter(Receiver.name == firstseen_null_query.columns.name) \ - .update({'firstseen': firstseen_null_query.columns.firstseen}) - - # update country code if None - unknown_country_query = app.session.query(Receiver) \ - .filter(Receiver.country_code == null()) \ - .filter(Receiver.location_wkt != null()) \ - .order_by(Receiver.name) - - for receiver in unknown_country_query.all(): - location = receiver.location - country_code = get_country_code(location.latitude, location.longitude) - if country_code is not None: - receiver.country_code = country_code - logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) - - logger.info("Added: {}, location changed: {}".format(added_count, changed_count)) - - app.session.commit() - - return update_count From ad05de59e8973fa76bc161736bda7068e364a015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 20:17:40 +0100 Subject: [PATCH 19/32] Reverted persistent dstcall... --- ogn/model/beacon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index 4ed46d3..d36dbe5 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -16,7 +16,7 @@ class Beacon(AbstractConcreteBase, Base): name = Column(String) receiver_name = Column(String(9)) - dstcall = Column(String) + dstcall = None timestamp = Column(DateTime, index=True) symboltable = None symbolcode = None From 966b7d9c734c4dd6d8c17592c5174f57fc596ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Fri, 8 Dec 2017 21:32:50 +0100 Subject: [PATCH 20/32] flake8 refactoring --- ogn/collect/database.py | 119 +++++++++++++++++++++------------------ ogn/commands/database.py | 13 +++-- 2 files changed, 70 insertions(+), 62 deletions(-) diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 7a99968..dcb3f7c 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -65,7 +65,8 @@ def update_devices(): app.session.commit() # For each address in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = app.session.query(distinct(AircraftBeacon.address).label('address'), + last_valid_values = app.session.query( + distinct(AircraftBeacon.address).label('address'), func.first_value(AircraftBeacon.timestamp) .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.timestamp == null(), None), (AircraftBeacon.timestamp != null(), AircraftBeacon.timestamp)])) .label('firstseen'), @@ -87,44 +88,47 @@ def update_devices(): func.first_value(AircraftBeacon.real_address) .over(partition_by=AircraftBeacon.address, order_by=case([(AircraftBeacon.real_address == null(), None), (AircraftBeacon.real_address != null(), AircraftBeacon.real_address)])) .label('real_address')) \ - .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ - .subquery() + .filter(and_(AircraftBeacon.device_id == null(), AircraftBeacon.error_count == 0)) \ + .subquery() - update_values = app.session.query(Device.address, - case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), - case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), - case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), - (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), - case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), - (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), - case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), - case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), - (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), - case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), - (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ - .filter(Device.address == last_valid_values.c.address) \ - .subquery() + update_values = app.session.query( + Device.address, + case([(or_(Device.firstseen == null(), Device.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Device.firstseen <= last_valid_values.c.firstseen, Device.firstseen)]).label('firstseen'), + case([(or_(Device.lastseen == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Device.lastseen >= last_valid_values.c.lastseen, Device.lastseen)]).label('lastseen'), + case([(or_(Device.aircraft_type == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.aircraft_type), + (Device.lastseen >= last_valid_values.c.lastseen, Device.aircraft_type)]).label('aircraft_type'), + case([(or_(Device.stealth == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.stealth), + (Device.lastseen >= last_valid_values.c.lastseen, Device.stealth)]).label('stealth'), + case([(or_(Device.software_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.software_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.software_version)]).label('software_version'), + case([(or_(Device.hardware_version == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.hardware_version), + (Device.lastseen >= last_valid_values.c.lastseen, Device.hardware_version)]).label('hardware_version'), + case([(or_(Device.real_address == null(), Device.lastseen < last_valid_values.c.lastseen), last_valid_values.c.real_address), + (Device.lastseen >= last_valid_values.c.lastseen, Device.real_address)]).label('real_address')) \ + .filter(Device.address == last_valid_values.c.address) \ + .subquery() update_receivers = app.session.query(Device) \ .filter(Device.address == update_values.c.address) \ - .update({Device.firstseen: update_values.c.firstseen, - Device.lastseen: update_values.c.lastseen, - Device.aircraft_type: update_values.c.aircraft_type, - Device.stealth: update_values.c.stealth, - Device.software_version: update_values.c.software_version, - Device.hardware_version: update_values.c.hardware_version, - Device.real_address: update_values.c.real_address}, - synchronize_session='fetch') + .update({ + Device.firstseen: update_values.c.firstseen, + Device.lastseen: update_values.c.lastseen, + Device.aircraft_type: update_values.c.aircraft_type, + Device.stealth: update_values.c.stealth, + Device.software_version: update_values.c.software_version, + Device.hardware_version: update_values.c.hardware_version, + Device.real_address: update_values.c.real_address}, + synchronize_session='fetch') # Update relations to aircraft beacons upd = app.session.query(AircraftBeacon) \ .filter(AircraftBeacon.device_id == null()) \ .filter(AircraftBeacon.address == Device.address) \ - .update({AircraftBeacon.device_id: Device.id}, - synchronize_session='fetch') + .update({ + AircraftBeacon.device_id: Device.id}, + synchronize_session='fetch') app.session.commit() print("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) @@ -147,7 +151,8 @@ def update_receivers(): insert_count = res.rowcount # For each name in the new beacons: get firstseen, lastseen and last values != NULL - last_valid_values = app.session.query(distinct(ReceiverBeacon.name).label('name'), + last_valid_values = app.session.query( + distinct(ReceiverBeacon.name).label('name'), func.first_value(ReceiverBeacon.timestamp) .over(partition_by=ReceiverBeacon.name, order_by=case([(ReceiverBeacon.timestamp == null(), None), (ReceiverBeacon.timestamp != null(), ReceiverBeacon.timestamp)])) .label('firstseen'), @@ -169,34 +174,36 @@ def update_receivers(): .filter(ReceiverBeacon.receiver_id == null()) \ .subquery() - update_values = app.session.query(Receiver.name, - case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), - (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), - (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), - (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), - case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), - (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), - case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed - (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ - .filter(Receiver.name == last_valid_values.c.name) \ - .subquery() + update_values = app.session.query( + Receiver.name, + case([(or_(Receiver.firstseen == null(), Receiver.firstseen > last_valid_values.c.firstseen), last_valid_values.c.firstseen), + (Receiver.firstseen <= last_valid_values.c.firstseen, Receiver.firstseen)]).label('firstseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.lastseen), + (Receiver.firstseen >= last_valid_values.c.firstseen, Receiver.firstseen)]).label('lastseen'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), func.ST_Transform(last_valid_values.c.location_wkt, 4326)), + (Receiver.lastseen >= last_valid_values.c.lastseen, func.ST_Transform(Receiver.location_wkt, 4326))]).label('location_wkt'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.altitude), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.altitude)]).label('altitude'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.version), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.version)]).label('version'), + case([(or_(Receiver.lastseen == null(), Receiver.lastseen < last_valid_values.c.lastseen), last_valid_values.c.platform), + (Receiver.lastseen >= last_valid_values.c.lastseen, Receiver.platform)]).label('platform'), + case([(or_(Receiver.location_wkt == null(), not_(func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt))), None), # set country code to None if location changed + (func.ST_Equals(Receiver.location_wkt, last_valid_values.c.location_wkt), Receiver.country_code)]).label('country_code')) \ + .filter(Receiver.name == last_valid_values.c.name) \ + .subquery() update_receivers = app.session.query(Receiver) \ .filter(Receiver.name == update_values.c.name) \ - .update({Receiver.firstseen: update_values.c.firstseen, - Receiver.lastseen: update_values.c.lastseen, - Receiver.location_wkt: update_values.c.location_wkt, - Receiver.altitude: update_values.c.altitude, - Receiver.version: update_values.c.version, - Receiver.platform: update_values.c.platform, - Receiver.country_code: update_values.c.country_code}, - synchronize_session='fetch') + .update({ + Receiver.firstseen: update_values.c.firstseen, + Receiver.lastseen: update_values.c.lastseen, + Receiver.location_wkt: update_values.c.location_wkt, + Receiver.altitude: update_values.c.altitude, + Receiver.version: update_values.c.version, + Receiver.platform: update_values.c.platform, + Receiver.country_code: update_values.c.country_code}, + synchronize_session='fetch') # Update relations to aircraft beacons update_aircraft_beacons = app.session.query(AircraftBeacon) \ diff --git a/ogn/commands/database.py b/ogn/commands/database.py index f01cd98..21d8a5b 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -93,12 +93,13 @@ def update_receivers(): def update_receiver_stats(): """Add/update entries in receiver stats table.""" - asdf = session.query(ReceiverBeacon.receiver_id, - func.count(distinct(AircraftBeacon.device_id)).label('device_count'), - func.max(AircraftBeacon.altitude).label('max_altitude'), - func.max(func.ST_Distance(AircraftBeacon.location_wkt, AircraftBeacon.location_wkt)).label('max_distance')) \ - .filter(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id) \ - .group_by(ReceiverBeacon.id) + asdf = session.query( + ReceiverBeacon.receiver_id, + func.count(distinct(AircraftBeacon.device_id)).label('device_count'), + func.max(AircraftBeacon.altitude).label('max_altitude'), + func.max(func.ST_Distance(AircraftBeacon.location_wkt, AircraftBeacon.location_wkt)).label('max_distance')) \ + .filter(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id) \ + .group_by(ReceiverBeacon.id) print(asdf) for a in asdf.all(): From 28a25048f6e434674921566f4aaed878865e1fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sun, 10 Dec 2017 17:30:27 +0100 Subject: [PATCH 21/32] Refactoring --- ogn/collect/database.py | 10 +- ogn/collect/logbook.py | 2 +- ogn/collect/takeoff_landing.py | 129 ++++++++++++++++---------- ogn/commands/logbook.py | 8 +- tests/collect/test_logbook.py | 34 +++---- tests/collect/test_takeoff_landing.py | 11 ++- 6 files changed, 115 insertions(+), 79 deletions(-) diff --git a/ogn/collect/database.py b/ogn/collect/database.py index dcb3f7c..dc551c6 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -131,8 +131,8 @@ def update_devices(): synchronize_session='fetch') app.session.commit() - print("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) - print("Updated {} AircraftBeacons".format(upd)) + logger.info("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) + logger.info("Updated {} AircraftBeacons".format(upd)) @app.task @@ -220,13 +220,13 @@ def update_receivers(): app.session.commit() - print("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) - print("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) + logger.info("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) + logger.info("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) @app.task def update_country_code(): - # update country code if None + # update country code in receivers table if None unknown_country_query = app.session.query(Receiver) \ .filter(Receiver.country_code == null()) \ .filter(Receiver.location_wkt != null()) \ diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index d9af45d..ff938a5 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -11,7 +11,7 @@ logger = get_task_logger(__name__) @app.task -def compute_logbook_entries(session=None): +def update_logbook(session=None): logger.info("Compute logbook.") if session is None: diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index e4572cd..48c0007 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -5,6 +5,7 @@ from celery.utils.log import get_task_logger from sqlalchemy import and_, or_, insert, between, exists from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import case +from sqlalchemy.orm import aliased from ogn.collect.celery import app from ogn.model import AircraftBeacon, TakeoffLanding, Airport @@ -13,7 +14,7 @@ logger = get_task_logger(__name__) @app.task -def compute_takeoff_and_landing(session=None): +def update_takeoff_landing(session=None): logger.info("Compute takeoffs and landings.") if session is None: @@ -43,73 +44,94 @@ def compute_takeoff_and_landing(session=None): # make a query with current, previous and next position sq = session.query( AircraftBeacon.id, - AircraftBeacon.timestamp, - func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'), - func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'), - AircraftBeacon.location_wkt, - func.lag(AircraftBeacon.location_wkt).over(order_by=wo).label('location_wkt_prev'), - func.lead(AircraftBeacon.location_wkt).over(order_by=wo).label('location_wkt_next'), - AircraftBeacon.track, - func.lag(AircraftBeacon.track).over(order_by=wo).label('track_prev'), - func.lead(AircraftBeacon.track).over(order_by=wo).label('track_next'), - AircraftBeacon.ground_speed, - func.lag(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_prev'), - func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'), - AircraftBeacon.altitude, - func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'), - func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next'), + func.lag(AircraftBeacon.id).over(order_by=wo).label('id_prev'), + func.lead(AircraftBeacon.id).over(order_by=wo).label('id_next'), AircraftBeacon.device_id, func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'), func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next')) \ .filter(AircraftBeacon.status == null()) \ + .filter(and_(AircraftBeacon.timestamp >= '2017-12-09 11:00:00', AircraftBeacon.timestamp <= '2017-12-09 18:00:00')) \ .subquery() + # consider only positions with the same device id sq2 = session.query(sq) \ - .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ + .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ + .subquery() + + print(sq2) + return + + # Get timestamps, locations, tracks, ground_speeds and altitudes + prev_ab = aliased(AircraftBeacon, name="prev_ab") + lead_ab = aliased(AircraftBeacon, name="lead_ab") + + sq3 = session.query( + sq2.c.id, + sq2.c.id_prev, + sq2.c.id_next, + sq2.c.device_id, + sq2.c.device_id_prev, + sq2.c.device_id_next, + AircraftBeacon.timestamp, + prev_ab.timestamp.label('timestamp_prev'), + lead_ab.timestamp.label('timestamp_next'), + AircraftBeacon.location_wkt, + prev_ab.location_wkt.label('location_wkt_prev'), + lead_ab.location_wkt.label('location_wkt_next'), + AircraftBeacon.track, + prev_ab.track.label('track_prev'), + lead_ab.track.label('track_next'), + AircraftBeacon.ground_speed, + prev_ab.ground_speed.label('ground_speed_prev'), + lead_ab.ground_speed.label('ground_speed_next'), + AircraftBeacon.altitude, + prev_ab.altitude.label('altitude_prev'), + lead_ab.altitude.label('altitude_next')) \ + .filter(and_(sq2.c.id == AircraftBeacon.id, sq2.c.id_prev == prev_ab.id, sq2.c.id_next == lead_ab.id)) \ .subquery() # find possible takeoffs and landings - sq3 = session.query( - sq2.c.id, - sq2.c.timestamp, - case([(sq2.c.ground_speed > takeoff_speed, sq2.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport - (sq2.c.ground_speed < landing_speed, sq2.c.location)]).label('location'), - case([(sq2.c.ground_speed > takeoff_speed, sq2.c.track), - (sq2.c.ground_speed < landing_speed, sq2.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly - sq2.c.ground_speed, - sq2.c.altitude, - case([(sq2.c.ground_speed > takeoff_speed, True), - (sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'), - sq2.c.device_id) \ - .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \ - .filter(and_(func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_prev, radius), - func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_next, radius))) \ - .filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff - sq2.c.ground_speed > takeoff_speed, - sq2.c.ground_speed_next > takeoff_speed), - and_(sq2.c.ground_speed_prev > landing_speed, # landing - sq2.c.ground_speed < landing_speed, - sq2.c.ground_speed_next < landing_speed))) \ + sq4 = session.query( + sq3.c.id, + sq3.c.timestamp, + case([(sq3.c.ground_speed > takeoff_speed, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport + (sq3.c.ground_speed < landing_speed, sq3.c.location)]).label('location'), + case([(sq3.c.ground_speed > takeoff_speed, sq3.c.track), + (sq3.c.ground_speed < landing_speed, sq3.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly + sq3.c.ground_speed, + sq3.c.altitude, + case([(sq3.c.ground_speed > takeoff_speed, True), + (sq3.c.ground_speed < landing_speed, False)]).label('is_takeoff'), + sq3.c.device_id) \ + .filter(sq3.c.timestamp_next - sq3.c.timestamp_prev < timedelta(seconds=duration)) \ + .filter(and_(func.ST_DFullyWithin(sq3.c.location, sq3.c.location_wkt_prev, radius), + func.ST_DFullyWithin(sq3.c.location, sq3.c.location_wkt_next, radius))) \ + .filter(or_(and_(sq3.c.ground_speed_prev < takeoff_speed, # takeoff + sq3.c.ground_speed > takeoff_speed, + sq3.c.ground_speed_next > takeoff_speed), + and_(sq3.c.ground_speed_prev > landing_speed, # landing + sq3.c.ground_speed < landing_speed, + sq3.c.ground_speed_next < landing_speed))) \ .subquery() # consider them if they are near a airport - sq4 = session.query( - sq3.c.timestamp, - sq3.c.track, - sq3.c.is_takeoff, - sq3.c.device_id, + sq5 = session.query( + sq4.c.timestamp, + sq4.c.track, + sq4.c.is_takeoff, + sq4.c.device_id, Airport.id.label('airport_id')) \ - .filter(and_(func.ST_DFullyWithin(sq3.c.location, Airport.location_wkt, airport_radius), - between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ + .filter(and_(func.ST_DFullyWithin(sq4.c.location, Airport.location_wkt, airport_radius), + between(sq4.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ .filter(between(Airport.style, 2, 5)) \ .subquery() # consider them only if they are not already existing in db - takeoff_landing_query = session.query(sq4) \ + takeoff_landing_query = session.query(sq5) \ .filter(~exists().where( - and_(TakeoffLanding.timestamp == sq4.c.timestamp, - TakeoffLanding.device_id == sq4.c.device_id, - TakeoffLanding.airport_id == sq4.c.airport_id))) + and_(TakeoffLanding.timestamp == sq5.c.timestamp, + TakeoffLanding.device_id == sq5.c.device_id, + TakeoffLanding.airport_id == sq5.c.airport_id))) # ... and save them ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp, @@ -120,7 +142,14 @@ def compute_takeoff_and_landing(session=None): takeoff_landing_query) result = session.execute(ins) counter = result.rowcount + + # mark the computated AircraftBeacons as 'used' + update_aircraft_beacons = session.query(AircraftBeacon) \ + .filter(AircraftBeacon.id == sq2.c.id) \ + .update({AircraftBeacon.status: 0}, + synchronize_session='fetch') + session.commit() - logger.debug("New takeoffs and landings: {}".format(counter)) + logger.debug("Inserted {} TakeoffLandings, updated {} AircraftBeacons".format(counter, update_aircraft_beacons)) return counter diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index a603933..8b33c55 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -3,8 +3,8 @@ from datetime import timedelta, datetime from manager import Manager -from ogn.collect.logbook import compute_logbook_entries -from ogn.collect.takeoff_landing import compute_takeoff_and_landing +from ogn.collect.logbook import update_logbook +from ogn.collect.takeoff_landing import update_takeoff_landing from ogn.commands.dbutils import session from ogn.model import Device, DeviceInfo, TakeoffLanding, Airport, Logbook from sqlalchemy import and_, or_ @@ -19,7 +19,7 @@ manager = Manager() def compute_takeoff_landing(): """Compute takeoffs and landings.""" print("Compute takeoffs and landings...") - result = compute_takeoff_and_landing.delay() + result = update_takeoff_landing.delay() counter = result.get() print("New takeoffs/landings: {}".format(counter)) @@ -28,7 +28,7 @@ def compute_takeoff_landing(): def compute_logbook(): """Compute logbook.""" print("Compute logbook...") - result = compute_logbook_entries.delay() + result = update_logbook.delay() counter = result.get() print("New logbook entries: {}".format(counter)) diff --git a/tests/collect/test_logbook.py b/tests/collect/test_logbook.py index 3836948..7aff8fc 100644 --- a/tests/collect/test_logbook.py +++ b/tests/collect/test_logbook.py @@ -1,7 +1,7 @@ import unittest import os -from ogn.collect.logbook import compute_logbook_entries +from ogn.collect.logbook import update_logbook class TestDB(unittest.TestCase): @@ -41,10 +41,10 @@ class TestDB(unittest.TestCase): session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_single_landing(self): @@ -53,10 +53,10 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_different_takeoffs(self): @@ -66,10 +66,10 @@ class TestDB(unittest.TestCase): session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/2') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_takeoff_and_landing(self): @@ -79,10 +79,10 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_takeoff_and_landing_on_different_days(self): @@ -92,10 +92,10 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815_LATER) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/2') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_update(self): @@ -104,22 +104,22 @@ class TestDB(unittest.TestCase): session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '1/0') session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/0') def test_update_wrong_order(self): @@ -128,13 +128,13 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '0/1') session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = compute_logbook_entries(session) + entries_changed = update_logbook(session) self.assertEqual(entries_changed, '1/0') diff --git a/tests/collect/test_takeoff_landing.py b/tests/collect/test_takeoff_landing.py index 4957621..d212ca6 100644 --- a/tests/collect/test_takeoff_landing.py +++ b/tests/collect/test_takeoff_landing.py @@ -3,7 +3,7 @@ import os from ogn.model import TakeoffLanding -from ogn.collect.takeoff_landing import compute_takeoff_and_landing +from ogn.collect.takeoff_landing import update_takeoff_landing class TestDB(unittest.TestCase): @@ -30,6 +30,7 @@ class TestDB(unittest.TestCase): session = self.session session.execute("DELETE FROM takeoff_landing") session.execute("DELETE FROM aircraft_beacon") + session.execute("DELETE FROM airport") session.commit() pass @@ -44,6 +45,7 @@ class TestDB(unittest.TestCase): return i def test_broken_rope(self): + """Fill the db with a winch launch where the rope breaks. The algorithm should detect one takeoff and one landing.""" session = self.session session.execute("INSERT INTO aircraft_beacon(address, location, altitude, timestamp, track, ground_speed, climb_rate, turn_rate) VALUES('DDEFF7','0101000020E61000009668B61829F12640330E0887F1E94740',604,'2016-07-02 10:47:12',0,0,0,0)") @@ -95,7 +97,12 @@ class TestDB(unittest.TestCase): session.execute("UPDATE aircraft_beacon SET device_id = d.id FROM device d WHERE d.address='DDEFF7'") session.commit() - compute_takeoff_and_landing(session) + # find the takeoff and the landing + update_takeoff_landing(session) + self.assertEqual(self.count_takeoff_and_landings(), 2) + + # we should not find the takeoff and the landing again + update_takeoff_landing(session) self.assertEqual(self.count_takeoff_and_landings(), 2) From 3956a11aa581ef5a067517528eff19e2aaa78785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sun, 10 Dec 2017 20:07:37 +0100 Subject: [PATCH 22/32] Refactoring --- ogn/collect/logbook.py | 34 +++++++++++++++++++++++++++++++++- ogn/collect/takeoff_landing.py | 12 +++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index ff938a5..e99fef9 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -5,7 +5,7 @@ from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import true, false from ogn.collect.celery import app -from ogn.model import TakeoffLanding, Logbook +from ogn.model import TakeoffLanding, Logbook, AircraftBeacon logger = get_task_logger(__name__) @@ -157,3 +157,35 @@ def update_logbook(session=None): logger.debug("New logbook entries: {}".format(insert_counter)) return "{}/{}".format(update_counter, insert_counter) + + +@app.task +def update_max_altitude(session=None): + logger.info("Update logbook max altitude.") + + if session is None: + session = app.session + + logbook_entries = session.query(Logbook.id) \ + .filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null())) \ + .limit(1000) \ + .subquery() + + max_altitudes = session.query(Logbook.id, func.max(AircraftBeacon.altitude).label('max_altitude')) \ + .filter(Logbook.id == logbook_entries.c.id) \ + .filter(and_(AircraftBeacon.device_id == Logbook.device_id, + AircraftBeacon.timestamp >= Logbook.takeoff_timestamp, + AircraftBeacon.timestamp <= Logbook.landing_timestamp)) \ + .group_by(Logbook.id) \ + .subquery() + + update_logbook = app.session.query(Logbook) \ + .filter(Logbook.id == max_altitudes.c.id) \ + .update({ + Logbook.max_altitude: max_altitudes.c.max_altitude}, + synchronize_session='fetch') + + session.commit() + logger.info("Logbook: {} entries updated.".format(update_logbook)) + + return update_logbook diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index 48c0007..26f53b6 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -42,6 +42,12 @@ def update_takeoff_landing(session=None): AircraftBeacon.receiver_id) # make a query with current, previous and next position + beacon_selection = session.query(AircraftBeacon.id) \ + .filter(AircraftBeacon.status == null()) \ + .order_by(AircraftBeacon.timestamp) \ + .limit(1000000) \ + .subquery() + sq = session.query( AircraftBeacon.id, func.lag(AircraftBeacon.id).over(order_by=wo).label('id_prev'), @@ -49,8 +55,7 @@ def update_takeoff_landing(session=None): AircraftBeacon.device_id, func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'), func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next')) \ - .filter(AircraftBeacon.status == null()) \ - .filter(and_(AircraftBeacon.timestamp >= '2017-12-09 11:00:00', AircraftBeacon.timestamp <= '2017-12-09 18:00:00')) \ + .filter(AircraftBeacon.id == beacon_selection.c.id) \ .subquery() # consider only positions with the same device id @@ -58,9 +63,6 @@ def update_takeoff_landing(session=None): .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ .subquery() - print(sq2) - return - # Get timestamps, locations, tracks, ground_speeds and altitudes prev_ab = aliased(AircraftBeacon, name="prev_ab") lead_ab = aliased(AircraftBeacon, name="lead_ab") From 5a31d902ab84c12a87b578e3ee5ff566d46875ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Mon, 11 Dec 2017 19:16:03 +0100 Subject: [PATCH 23/32] Merged window function, this is 10% faster and looks better --- ogn/collect/takeoff_landing.py | 108 ++++++++++++++------------------- 1 file changed, 47 insertions(+), 61 deletions(-) diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index 26f53b6..539e60c 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -54,7 +54,22 @@ def update_takeoff_landing(session=None): func.lead(AircraftBeacon.id).over(order_by=wo).label('id_next'), AircraftBeacon.device_id, func.lag(AircraftBeacon.device_id).over(order_by=wo).label('device_id_prev'), - func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next')) \ + func.lead(AircraftBeacon.device_id).over(order_by=wo).label('device_id_next'), + AircraftBeacon.timestamp, + func.lag(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_prev'), + func.lead(AircraftBeacon.timestamp).over(order_by=wo).label('timestamp_next'), + AircraftBeacon.location_wkt, + func.lag(AircraftBeacon.location_wkt).over(order_by=wo).label('location_wkt_prev'), + func.lead(AircraftBeacon.location_wkt).over(order_by=wo).label('location_wkt_next'), + AircraftBeacon.track, + func.lag(AircraftBeacon.track).over(order_by=wo).label('track_prev'), + func.lead(AircraftBeacon.track).over(order_by=wo).label('track_next'), + AircraftBeacon.ground_speed, + func.lag(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_prev'), + func.lead(AircraftBeacon.ground_speed).over(order_by=wo).label('ground_speed_next'), + AircraftBeacon.altitude, + func.lag(AircraftBeacon.altitude).over(order_by=wo).label('altitude_prev'), + func.lead(AircraftBeacon.altitude).over(order_by=wo).label('altitude_next')) \ .filter(AircraftBeacon.id == beacon_selection.c.id) \ .subquery() @@ -63,77 +78,48 @@ def update_takeoff_landing(session=None): .filter(sq.c.device_id_prev == sq.c.device_id == sq.c.device_id_next) \ .subquery() - # Get timestamps, locations, tracks, ground_speeds and altitudes - prev_ab = aliased(AircraftBeacon, name="prev_ab") - lead_ab = aliased(AircraftBeacon, name="lead_ab") - + # find possible takeoffs and landings sq3 = session.query( sq2.c.id, - sq2.c.id_prev, - sq2.c.id_next, - sq2.c.device_id, - sq2.c.device_id_prev, - sq2.c.device_id_next, - AircraftBeacon.timestamp, - prev_ab.timestamp.label('timestamp_prev'), - lead_ab.timestamp.label('timestamp_next'), - AircraftBeacon.location_wkt, - prev_ab.location_wkt.label('location_wkt_prev'), - lead_ab.location_wkt.label('location_wkt_next'), - AircraftBeacon.track, - prev_ab.track.label('track_prev'), - lead_ab.track.label('track_next'), - AircraftBeacon.ground_speed, - prev_ab.ground_speed.label('ground_speed_prev'), - lead_ab.ground_speed.label('ground_speed_next'), - AircraftBeacon.altitude, - prev_ab.altitude.label('altitude_prev'), - lead_ab.altitude.label('altitude_next')) \ - .filter(and_(sq2.c.id == AircraftBeacon.id, sq2.c.id_prev == prev_ab.id, sq2.c.id_next == lead_ab.id)) \ - .subquery() - - # find possible takeoffs and landings - sq4 = session.query( - sq3.c.id, - sq3.c.timestamp, - case([(sq3.c.ground_speed > takeoff_speed, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport - (sq3.c.ground_speed < landing_speed, sq3.c.location)]).label('location'), - case([(sq3.c.ground_speed > takeoff_speed, sq3.c.track), - (sq3.c.ground_speed < landing_speed, sq3.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly - sq3.c.ground_speed, - sq3.c.altitude, - case([(sq3.c.ground_speed > takeoff_speed, True), - (sq3.c.ground_speed < landing_speed, False)]).label('is_takeoff'), - sq3.c.device_id) \ - .filter(sq3.c.timestamp_next - sq3.c.timestamp_prev < timedelta(seconds=duration)) \ - .filter(and_(func.ST_DFullyWithin(sq3.c.location, sq3.c.location_wkt_prev, radius), - func.ST_DFullyWithin(sq3.c.location, sq3.c.location_wkt_next, radius))) \ - .filter(or_(and_(sq3.c.ground_speed_prev < takeoff_speed, # takeoff - sq3.c.ground_speed > takeoff_speed, - sq3.c.ground_speed_next > takeoff_speed), - and_(sq3.c.ground_speed_prev > landing_speed, # landing - sq3.c.ground_speed < landing_speed, - sq3.c.ground_speed_next < landing_speed))) \ + sq2.c.timestamp, + case([(sq2.c.ground_speed > takeoff_speed, sq2.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport + (sq2.c.ground_speed < landing_speed, sq2.c.location)]).label('location'), + case([(sq2.c.ground_speed > takeoff_speed, sq2.c.track), + (sq2.c.ground_speed < landing_speed, sq2.c.track_prev)]).label('track'), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly + sq2.c.ground_speed, + sq2.c.altitude, + case([(sq2.c.ground_speed > takeoff_speed, True), + (sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'), + sq2.c.device_id) \ + .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \ + .filter(and_(func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_prev, radius), + func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_next, radius))) \ + .filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff + sq2.c.ground_speed > takeoff_speed, + sq2.c.ground_speed_next > takeoff_speed), + and_(sq2.c.ground_speed_prev > landing_speed, # landing + sq2.c.ground_speed < landing_speed, + sq2.c.ground_speed_next < landing_speed))) \ .subquery() # consider them if they are near a airport - sq5 = session.query( - sq4.c.timestamp, - sq4.c.track, - sq4.c.is_takeoff, - sq4.c.device_id, + sq4 = session.query( + sq3.c.timestamp, + sq3.c.track, + sq3.c.is_takeoff, + sq3.c.device_id, Airport.id.label('airport_id')) \ - .filter(and_(func.ST_DFullyWithin(sq4.c.location, Airport.location_wkt, airport_radius), - between(sq4.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ + .filter(and_(func.ST_DFullyWithin(sq3.c.location, Airport.location_wkt, airport_radius), + between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ .filter(between(Airport.style, 2, 5)) \ .subquery() # consider them only if they are not already existing in db - takeoff_landing_query = session.query(sq5) \ + takeoff_landing_query = session.query(sq4) \ .filter(~exists().where( - and_(TakeoffLanding.timestamp == sq5.c.timestamp, - TakeoffLanding.device_id == sq5.c.device_id, - TakeoffLanding.airport_id == sq5.c.airport_id))) + and_(TakeoffLanding.timestamp == sq4.c.timestamp, + TakeoffLanding.device_id == sq4.c.device_id, + TakeoffLanding.airport_id == sq4.c.airport_id))) # ... and save them ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp, From 9a21f582efb23355929d817170b97ba27d2d23be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 03:47:28 +0100 Subject: [PATCH 24/32] Speedup distance calculation --- ogn/collect/takeoff_landing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index 539e60c..f2df2e8 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -30,10 +30,10 @@ def update_takeoff_landing(session=None): takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit duration = 100 # the points must not exceed this duration - radius = 0.05 # the points must not exceed this radius (degree!) around the 2nd point + radius = 5000 # the points must not exceed this radius around the 2nd point # takeoff / landing has to be near an airport - airport_radius = 0.025 # takeoff / landing must not exceed this radius (degree!) around the airport + airport_radius = 2500 # takeoff / landing must not exceed this radius around the airport airport_delta = 100 # takeoff / landing must not exceed this altitude offset above/below the airport # 'wo' is the window order for the sql window function @@ -92,8 +92,8 @@ def update_takeoff_landing(session=None): (sq2.c.ground_speed < landing_speed, False)]).label('is_takeoff'), sq2.c.device_id) \ .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) \ - .filter(and_(func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_prev, radius), - func.ST_DFullyWithin(sq2.c.location, sq2.c.location_wkt_next, radius))) \ + .filter(and_(func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_prev) < radius, + func.ST_Distance_Sphere(sq2.c.location, sq2.c.location_wkt_next) < radius)) \ .filter(or_(and_(sq2.c.ground_speed_prev < takeoff_speed, # takeoff sq2.c.ground_speed > takeoff_speed, sq2.c.ground_speed_next > takeoff_speed), @@ -109,7 +109,7 @@ def update_takeoff_landing(session=None): sq3.c.is_takeoff, sq3.c.device_id, Airport.id.label('airport_id')) \ - .filter(and_(func.ST_DFullyWithin(sq3.c.location, Airport.location_wkt, airport_radius), + .filter(and_(func.ST_Distance_Sphere(sq3.c.location, Airport.location_wkt) < airport_radius, between(sq3.c.altitude, Airport.altitude - airport_delta, Airport.altitude + airport_delta))) \ .filter(between(Airport.style, 2, 5)) \ .subquery() From 2cc3b8f71c7253d531d4e482b5c9457841f70b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 09:15:31 +0100 Subject: [PATCH 25/32] Better return values for tasks --- ogn/collect/database.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ogn/collect/database.py b/ogn/collect/database.py index dc551c6..3954bfe 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -134,6 +134,9 @@ def update_devices(): logger.info("Devices: {} inserted, {} updated".format(insert_count, update_receivers)) logger.info("Updated {} AircraftBeacons".format(upd)) + return "{} Devices inserted, {} Devices updated, {} AircraftBeacons updated" \ + .format(insert_count, update_receivers, upd) + @app.task def update_receivers(): @@ -223,6 +226,9 @@ def update_receivers(): logger.info("Receivers: {} inserted, {} updated.".format(insert_count, update_receivers)) logger.info("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) + return "{} Receivers inserted, {} Receivers updated, {} AircraftBeacons updated, {} ReceiverBeacons updated" \ + .format(insert_count, update_receivers, update_aircraft_beacons, update_receiver_beacons) + @app.task def update_country_code(): @@ -232,11 +238,15 @@ def update_country_code(): .filter(Receiver.location_wkt != null()) \ .order_by(Receiver.name) + counter = 0 for receiver in unknown_country_query.all(): location = receiver.location country_code = get_country_code(location.latitude, location.longitude) if country_code is not None: receiver.country_code = country_code - print("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + counter += 1 app.session.commit() + + return "Updated country_code for {} Receivers".format(counter) \ No newline at end of file From 145f21604ad52692cda4f28585e543b45290a6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 09:59:38 +0100 Subject: [PATCH 26/32] Better logbook testing --- ogn/collect/logbook.py | 6 +-- tests/collect/test_logbook.py | 93 +++++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index e99fef9..caf493b 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -17,9 +17,6 @@ def update_logbook(session=None): if session is None: session = app.session - or_args = [between(TakeoffLanding.timestamp, '2016-06-28 00:00:00', '2016-06-28 23:59:59')] - or_args = [] - # 'wo' is the window order for the sql window function wo = and_(func.date(TakeoffLanding.timestamp), TakeoffLanding.device_id, @@ -43,7 +40,6 @@ def update_logbook(session=None): TakeoffLanding.airport_id, func.lag(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_prev'), func.lead(TakeoffLanding.airport_id).over(order_by=wo).label('airport_id_next')) \ - .filter(*or_args) \ .subquery() # find complete flights (with takeoff and landing on the same day) @@ -156,7 +152,7 @@ def update_logbook(session=None): session.commit() logger.debug("New logbook entries: {}".format(insert_counter)) - return "{}/{}".format(update_counter, insert_counter) + return "Logbook entries: {} inserted, {} updated".format(update_counter, insert_counter) @app.task diff --git a/tests/collect/test_logbook.py b/tests/collect/test_logbook.py index 7aff8fc..9d979dc 100644 --- a/tests/collect/test_logbook.py +++ b/tests/collect/test_logbook.py @@ -1,6 +1,9 @@ import unittest import os +from sqlalchemy.sql import null, and_ + +from ogn.model import Logbook, Airport from ogn.collect.logbook import update_logbook @@ -35,17 +38,43 @@ class TestDB(unittest.TestCase): session.commit() pass + def count_logbook_entries(self): + session = self.session + query = session.query(Logbook) + return len(query.all()) + + def assert_entries(self, koen_to=0, koen_ldg=0, koen_complete=0, ohl_to=0, ohl_ldg=0, ohl_complete=0): + session = self.session + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.landing_airport_id == null()).all()) + self.assertEqual(entries, koen_to) + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == null()).all()) + self.assertEqual(entries, koen_ldg) + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Koenigsdorf')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all()) + self.assertEqual(entries, koen_complete) + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.landing_airport_id == null()).all()) + self.assertEqual(entries, ohl_to) + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.landing_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == null()).all()) + self.assertEqual(entries, ohl_ldg) + + entries = len(session.query(Logbook).filter(and_(Airport.id == Logbook.takeoff_airport_id, Airport.name == 'Ohlstadt')).filter(Logbook.takeoff_airport_id == Logbook.landing_airport_id).all()) + self.assertEqual(entries, ohl_complete) + def test_single_takeoff(self): session = self.session session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_to=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_to=1) def test_single_landing(self): session = self.session @@ -53,11 +82,11 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_ldg=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_ldg=1) def test_different_takeoffs(self): session = self.session @@ -66,11 +95,11 @@ class TestDB(unittest.TestCase): session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/2') + update_logbook(session) + self.assert_entries(koen_to=1, ohl_to=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_to=1, ohl_to=1) def test_takeoff_and_landing(self): session = self.session @@ -79,11 +108,11 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_complete=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_complete=1) def test_takeoff_and_landing_on_different_days(self): session = self.session @@ -92,11 +121,11 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815_LATER) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/2') + update_logbook(session) + self.assert_entries(koen_to=1, koen_ldg=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_to=1, koen_ldg=1) def test_update(self): session = self.session @@ -104,23 +133,23 @@ class TestDB(unittest.TestCase): session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_to=1) session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '1/0') + update_logbook(session) + self.assert_entries(koen_complete=1) session.execute(self.TAKEOFF_OHLSTADT_DD4711) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_complete=1, ohl_to=1) - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/0') + update_logbook(session) + self.assert_entries(koen_complete=1, ohl_to=1) def test_update_wrong_order(self): session = self.session @@ -128,14 +157,14 @@ class TestDB(unittest.TestCase): session.execute(self.LANDING_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '0/1') + update_logbook(session) + self.assert_entries(koen_ldg=1) session.execute(self.TAKEOFF_KOENIGSDF_DD0815) session.commit() - entries_changed = update_logbook(session) - self.assertEqual(entries_changed, '1/0') + update_logbook(session) + self.assert_entries(koen_complete=1) if __name__ == '__main__': From b87510b9b7c57399c3fda894041e6e65e942c4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 21:45:19 +0100 Subject: [PATCH 27/32] Use status == 1 for 'used for takeoff-landing-detection' instead of 0 --- ogn/collect/logbook.py | 2 +- ogn/collect/takeoff_landing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index caf493b..bd85778 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -184,4 +184,4 @@ def update_max_altitude(session=None): session.commit() logger.info("Logbook: {} entries updated.".format(update_logbook)) - return update_logbook + return "Logbook: {} entries updated.".format(update_logbook) diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index f2df2e8..fcfc739 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -134,7 +134,7 @@ def update_takeoff_landing(session=None): # mark the computated AircraftBeacons as 'used' update_aircraft_beacons = session.query(AircraftBeacon) \ .filter(AircraftBeacon.id == sq2.c.id) \ - .update({AircraftBeacon.status: 0}, + .update({AircraftBeacon.status: 1}, synchronize_session='fetch') session.commit() From 3847db4abdd08e2a6c160a8bcb66d615e10272b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 21:46:21 +0100 Subject: [PATCH 28/32] Added DeviceStats and ReceiverStats --- ogn/collect/celery.py | 1 + ogn/collect/stats.py | 90 +++++++++++++++++++++++++++++++++++++ ogn/model/__init__.py | 2 + ogn/model/device_stats.py | 21 +++++++++ ogn/model/receiver_stats.py | 22 +++++++++ 5 files changed, 136 insertions(+) create mode 100644 ogn/collect/stats.py create mode 100644 ogn/model/device_stats.py create mode 100644 ogn/model/receiver_stats.py diff --git a/ogn/collect/celery.py b/ogn/collect/celery.py index fca88a1..6abf880 100644 --- a/ogn/collect/celery.py +++ b/ogn/collect/celery.py @@ -27,6 +27,7 @@ def close_db(signal, sender): app = Celery('ogn.collect', include=["ogn.collect.database", "ogn.collect.logbook", + "ogn.collect.stats", "ogn.collect.takeoff_landing", ]) diff --git a/ogn/collect/stats.py b/ogn/collect/stats.py new file mode 100644 index 0000000..9b435c0 --- /dev/null +++ b/ogn/collect/stats.py @@ -0,0 +1,90 @@ +from celery.utils.log import get_task_logger + +from sqlalchemy import insert, distinct +from sqlalchemy.sql import null, and_, or_, func, not_ + +from ogn.model import AircraftBeacon, ReceiverBeacon, Device, Receiver, DeviceStats, ReceiverStats + +from .celery import app + +logger = get_task_logger(__name__) + + +@app.task +def update_device_stats(date=None): + """Add/update entries in device stats table.""" + + if not date: + logger.warn("A date is needed for calculating stats. Exiting") + return None + + # First kill the stats for the selected date + deleted_counter = app.session.query(DeviceStats) \ + .filter(DeviceStats.date == date) \ + .delete() + + # Calculate stats for the selected date + device_stats = app.session.query( + AircraftBeacon.device_id, + func.date(AircraftBeacon.timestamp).label('date'), + func.count(distinct(AircraftBeacon.receiver_id)).label('receiver_count'), + func.count(AircraftBeacon.id).label('aircraft_beacon_count'), + func.max(AircraftBeacon.altitude).label('max_altitude')) \ + .filter(and_(AircraftBeacon.device_id != null(), AircraftBeacon.receiver_id != null())) \ + .filter(func.date(AircraftBeacon.timestamp) == date) \ + .group_by(AircraftBeacon.device_id, func.date(AircraftBeacon.timestamp)) \ + .subquery() + + # And insert them + ins = insert(DeviceStats).from_select( + [DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.aircraft_beacon_count, DeviceStats.max_altitude], + device_stats) + res = app.session.execute(ins) + insert_counter = res.rowcount + app.session.commit() + logger.debug("DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) + + return "DeviceStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter) + + +@app.task +def update_receiver_stats(date=None): + """Add/update entries in receiver stats table.""" + + if not date: + logger.warn("A date is needed for calculating stats. Exiting") + return None + + return "Not usable. Need indices." + + # First kill the stats for the selected date + deleted_counter = app.session.query(ReceiverStats) \ + .filter(ReceiverStats.date == date) \ + .delete() + + # Calculate stats for the selected date + receiver_stats = app.session.query( + ReceiverBeacon.receiver_id, + func.date(ReceiverBeacon.timestamp).label('date'), + func.count(AircraftBeacon.id).label('aircraft_beacon_count'), + func.count(ReceiverBeacon.id).label('receiver_beacon_count'), + func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'), + func.max(AircraftBeacon.distance).label('max_distance')) \ + .filter(and_(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id, ReceiverBeacon.receiver_id != null(), AircraftBeacon.receiver_id != null())) \ + .filter(and_(func.date(ReceiverBeacon.timestamp) == date, func.date(AircraftBeacon.timestamp) == date)) \ + .group_by(ReceiverBeacon.receiver_id, func.date(ReceiverBeacon.timestamp)) \ + .subquery() + + print(receiver_stats) + return + + # And insert them + ins = insert(ReceiverStats).from_select( + [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.receiver_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance], + receiver_stats) + res = app.session.execute(ins) + insert_counter = res.rowcount + app.session.commit() + logger.debug("ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) + + return "ReceiverStats entries for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter) diff --git a/ogn/model/__init__.py b/ogn/model/__init__.py index 42f8385..57c42c4 100644 --- a/ogn/model/__init__.py +++ b/ogn/model/__init__.py @@ -5,9 +5,11 @@ from .beacon import Beacon from .device import Device from .device_info import DeviceInfo from .device_info_origin import DeviceInfoOrigin +from .device_stats import DeviceStats from .aircraft_beacon import AircraftBeacon from .receiver_beacon import ReceiverBeacon from .receiver import Receiver +from .receiver_stats import ReceiverStats from .takeoff_landing import TakeoffLanding from .airport import Airport from .logbook import Logbook diff --git a/ogn/model/device_stats.py b/ogn/model/device_stats.py new file mode 100644 index 0000000..c8748de --- /dev/null +++ b/ogn/model/device_stats.py @@ -0,0 +1,21 @@ +from geoalchemy2.shape import to_shape +from geoalchemy2.types import Geometry +from sqlalchemy import Column, String, Integer, Date, Float, ForeignKey +from sqlalchemy.orm import relationship + +from .base import Base + + +class DeviceStats(Base): + __tablename__ = "device_stats" + + id = Column(Integer, primary_key=True) + + date = Column(Date) + receiver_count = Column(Integer) + aircraft_beacon_count = Column(Integer) + max_altitude = Column(Float) + + # Relations + device_id = Column(Integer, ForeignKey('device.id', ondelete='SET NULL'), index=True) + device = relationship('Device', foreign_keys=[device_id]) diff --git a/ogn/model/receiver_stats.py b/ogn/model/receiver_stats.py new file mode 100644 index 0000000..2065855 --- /dev/null +++ b/ogn/model/receiver_stats.py @@ -0,0 +1,22 @@ +from geoalchemy2.shape import to_shape +from geoalchemy2.types import Geometry +from sqlalchemy import Column, String, Integer, Date, Float, ForeignKey +from sqlalchemy.orm import relationship + +from .base import Base + + +class ReceiverStats(Base): + __tablename__ = "receiver_stats" + + id = Column(Integer, primary_key=True) + + date = Column(Date) + aircraft_beacon_count = Column(Integer) + receiver_beacon_count = Column(Integer) + aircraft_count = Column(Integer) + max_distance = Column(Float) + + # Relations + receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True) + receiver = relationship('Receiver', foreign_keys=[receiver_id]) From 58f9d543428437c554fc2c45497c3bf55ecf513a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 23:34:31 +0100 Subject: [PATCH 29/32] Removed obsolete code --- ogn/commands/database.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 21d8a5b..ca1dd20 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -81,31 +81,3 @@ def import_airports(path='tests/SeeYou.cup'): session.bulk_save_objects(airports) session.commit() print("Imported {} airports.".format(len(airports))) - - -@manager.command -def update_receivers(): - from ogn.collect.database import update_receivers as ur - ur() - - -@manager.command -def update_receiver_stats(): - """Add/update entries in receiver stats table.""" - - asdf = session.query( - ReceiverBeacon.receiver_id, - func.count(distinct(AircraftBeacon.device_id)).label('device_count'), - func.max(AircraftBeacon.altitude).label('max_altitude'), - func.max(func.ST_Distance(AircraftBeacon.location_wkt, AircraftBeacon.location_wkt)).label('max_distance')) \ - .filter(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id) \ - .group_by(ReceiverBeacon.id) - - print(asdf) - for a in asdf.all(): - print(a) - - return - - asdf = session.query(distinct(ReceiverBeacon.receiver_id), func.DATE(ReceiverBeacon.timestamp).label('date')) \ - .filter(ReceiverBeacon.receiver_id != null()) From 0b36569b0ba25886ba3e457bbbced4101310b2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Tue, 12 Dec 2017 23:35:39 +0100 Subject: [PATCH 30/32] Make ReceiverStats working without receiver_beacon_count --- ogn/collect/stats.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/ogn/collect/stats.py b/ogn/collect/stats.py index 9b435c0..d485987 100644 --- a/ogn/collect/stats.py +++ b/ogn/collect/stats.py @@ -2,6 +2,7 @@ from celery.utils.log import get_task_logger from sqlalchemy import insert, distinct from sqlalchemy.sql import null, and_, or_, func, not_ +from sqlalchemy.sql.expression import literal_column from ogn.model import AircraftBeacon, ReceiverBeacon, Device, Receiver, DeviceStats, ReceiverStats @@ -55,8 +56,6 @@ def update_receiver_stats(date=None): logger.warn("A date is needed for calculating stats. Exiting") return None - return "Not usable. Need indices." - # First kill the stats for the selected date deleted_counter = app.session.query(ReceiverStats) \ .filter(ReceiverStats.date == date) \ @@ -64,23 +63,19 @@ def update_receiver_stats(date=None): # Calculate stats for the selected date receiver_stats = app.session.query( - ReceiverBeacon.receiver_id, - func.date(ReceiverBeacon.timestamp).label('date'), + AircraftBeacon.receiver_id, + literal_column("'{}'".format(date)).label('date'), func.count(AircraftBeacon.id).label('aircraft_beacon_count'), - func.count(ReceiverBeacon.id).label('receiver_beacon_count'), func.count(distinct(AircraftBeacon.device_id)).label('aircraft_count'), func.max(AircraftBeacon.distance).label('max_distance')) \ - .filter(and_(ReceiverBeacon.receiver_id == AircraftBeacon.receiver_id, ReceiverBeacon.receiver_id != null(), AircraftBeacon.receiver_id != null())) \ - .filter(and_(func.date(ReceiverBeacon.timestamp) == date, func.date(AircraftBeacon.timestamp) == date)) \ - .group_by(ReceiverBeacon.receiver_id, func.date(ReceiverBeacon.timestamp)) \ + .filter(AircraftBeacon.receiver_id != null()) \ + .filter(func.date(AircraftBeacon.timestamp) == date) \ + .group_by(AircraftBeacon.receiver_id) \ .subquery() - print(receiver_stats) - return - # And insert them ins = insert(ReceiverStats).from_select( - [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.receiver_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance], + [ReceiverStats.receiver_id, ReceiverStats.date, ReceiverStats.aircraft_beacon_count, ReceiverStats.aircraft_count, ReceiverStats.max_distance], receiver_stats) res = app.session.execute(ins) insert_counter = res.rowcount From b812b9b38f6eaa6a087064f9c52831b64e2cd29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 13 Dec 2017 10:45:43 +0100 Subject: [PATCH 31/32] Show logbook with row_numbers --- ogn/commands/logbook.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index 8b33c55..a9c712d 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -63,7 +63,8 @@ def show(airport_name, utc_delta_hours=0, date=None): # get all logbook entries and add device and airport infos takeoff_airport = aliased(Airport, name='takeoff_airport') landing_airport = aliased(Airport, name='landing_airport') - logbook_query = session.query(Logbook, + logbook_query = session.query(func.row_number().over(order_by=Logbook.reftime).label('row_number'), + Logbook, Device, sq3.c.registration, sq3.c.aircraft) \ @@ -105,8 +106,9 @@ def show(airport_name, utc_delta_hours=0, date=None): def none_altitude_replacer(altitude_object, airport_object): return "?" if altitude_object is None else "{:5d}m ({:+5d}m)".format(altitude_object, altitude_object - airport_object.altitude) - for [logbook, device, registration, aircraft] in logbook_query.all(): - print('%10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % ( + for [row_number, logbook, device, registration, aircraft] in logbook_query.all(): + print('%3d. %10s %8s (%2s) %8s (%2s) %8s %15s %8s %17s %20s' % ( + row_number, logbook.reftime.date(), none_datetime_replacer(logbook.takeoff_timestamp), none_track_replacer(logbook.takeoff_track), From 99c07fed5bc088bab6ede488285b6cd9a9a70148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 13 Dec 2017 10:58:48 +0100 Subject: [PATCH 32/32] Fix travis build --- .travis.yml | 2 -- ogn/collect/database.py | 2 +- ogn/collect/logbook.py | 2 +- ogn/collect/stats.py | 4 ++-- ogn/collect/takeoff_landing.py | 1 - ogn/model/device_stats.py | 4 +--- ogn/model/receiver_stats.py | 4 +--- tests/gateway/test_manage.py | 23 ----------------------- 8 files changed, 6 insertions(+), 36 deletions(-) delete mode 100644 tests/gateway/test_manage.py diff --git a/.travis.yml b/.travis.yml index 71fa3f6..ebc54fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,11 +18,9 @@ before_script: - flake8 tests ogn_test - psql -U postgres -c 'CREATE DATABASE ogn_test;' - psql -U postgres -c 'CREATE EXTENSION postgis;' - - psql -U postgres -d ogn_test script: - nosetests --with-coverage --cover-package=ogn - - pip install . --upgrade - python -c 'import ogn' diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 3954bfe..f5c75ed 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -249,4 +249,4 @@ def update_country_code(): app.session.commit() - return "Updated country_code for {} Receivers".format(counter) \ No newline at end of file + return "Updated country_code for {} Receivers".format(counter) diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index bd85778..b086459 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -1,6 +1,6 @@ from celery.utils.log import get_task_logger -from sqlalchemy import and_, or_, insert, update, between, exists +from sqlalchemy import and_, or_, insert, update, exists from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import true, false diff --git a/ogn/collect/stats.py b/ogn/collect/stats.py index d485987..fe3a069 100644 --- a/ogn/collect/stats.py +++ b/ogn/collect/stats.py @@ -1,10 +1,10 @@ from celery.utils.log import get_task_logger from sqlalchemy import insert, distinct -from sqlalchemy.sql import null, and_, or_, func, not_ +from sqlalchemy.sql import null, and_, func from sqlalchemy.sql.expression import literal_column -from ogn.model import AircraftBeacon, ReceiverBeacon, Device, Receiver, DeviceStats, ReceiverStats +from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats from .celery import app diff --git a/ogn/collect/takeoff_landing.py b/ogn/collect/takeoff_landing.py index fcfc739..558a365 100644 --- a/ogn/collect/takeoff_landing.py +++ b/ogn/collect/takeoff_landing.py @@ -5,7 +5,6 @@ from celery.utils.log import get_task_logger from sqlalchemy import and_, or_, insert, between, exists from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import case -from sqlalchemy.orm import aliased from ogn.collect.celery import app from ogn.model import AircraftBeacon, TakeoffLanding, Airport diff --git a/ogn/model/device_stats.py b/ogn/model/device_stats.py index c8748de..b881ad4 100644 --- a/ogn/model/device_stats.py +++ b/ogn/model/device_stats.py @@ -1,6 +1,4 @@ -from geoalchemy2.shape import to_shape -from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, Date, Float, ForeignKey +from sqlalchemy import Column, Integer, Date, Float, ForeignKey from sqlalchemy.orm import relationship from .base import Base diff --git a/ogn/model/receiver_stats.py b/ogn/model/receiver_stats.py index 2065855..d24e311 100644 --- a/ogn/model/receiver_stats.py +++ b/ogn/model/receiver_stats.py @@ -1,6 +1,4 @@ -from geoalchemy2.shape import to_shape -from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, Date, Float, ForeignKey +from sqlalchemy import Column, Integer, Date, Float, ForeignKey from sqlalchemy.orm import relationship from .base import Base diff --git a/tests/gateway/test_manage.py b/tests/gateway/test_manage.py deleted file mode 100644 index b4a952d..0000000 --- a/tests/gateway/test_manage.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest -import unittest.mock as mock - -from ogn.gateway.manage import run -# from ogn.gateway.manage import import_logfile - - -class GatewayManagerTest(unittest.TestCase): - # try simple user interrupt - @mock.patch('ogn.gateway.manage.AprsClient') - def test_run_user_interruption(self, mock_aprs_client): - instance = mock_aprs_client.return_value - instance.run.side_effect = KeyboardInterrupt() - - run(aprs_user="testuser") - - instance.connect.assert_called_once_with() - self.assertEqual(instance.run.call_count, 1) - instance.disconnect.assert_called_once_with() - - -if __name__ == '__main__': - unittest.main()