From b135a9dfd0db8d987cee840e4de229d54497cf22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 3 Feb 2016 23:09:11 +0100 Subject: [PATCH 1/8] Update receivers --- config/default.py | 4 ++ ogn/collect/database.py | 87 +++++++++++++++++++++++++++++++++++- ogn/commands/showreceiver.py | 8 ++-- ogn/utils.py | 6 ++- 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/config/default.py b/config/default.py index f78c931..32ef2b1 100644 --- a/config/default.py +++ b/config/default.py @@ -11,6 +11,10 @@ CELERYBEAT_SCHEDULE = { 'task': 'ogn.collect.heatmap.update_beacon_receiver_distance_all', 'schedule': timedelta(minutes=5), }, + 'update-receiver-table': { + 'task': 'ogn.collect.database.update_receivers', + 'schedule': timedelta(minutes=5), + }, } CELERY_TIMEZONE = 'UTC' diff --git a/ogn/collect/database.py b/ogn/collect/database.py index b390795..b62ddff 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -1,9 +1,15 @@ -from ogn.utils import get_ddb -from ogn.model import Device, AddressOrigin +from sqlalchemy.sql import func, null +from sqlalchemy.sql.functions import coalesce +from sqlalchemy import and_, or_ from celery.utils.log import get_task_logger + +from ogn.model import Device, AddressOrigin, Receiver, ReceiverBeacon +from ogn.utils import get_ddb, get_country_code + from ogn.collect.celery import app + logger = get_task_logger(__name__) @@ -34,3 +40,80 @@ def import_file(path='tests/custom_ddb.txt'): logger.info("Import registered devices from '{}'...".format(path)) counter = update_devices(app.session, AddressOrigin.user_defined, get_ddb(path)) logger.info("Imported %i devices." % counter) + + +@app.task +def update_receivers(): + """Update the receiver table.""" + ## get current receiver data + last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ + .subquery() + + last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ + .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ + .group_by(ReceiverBeacon.name) \ + .subquery() + + ## update existing receivers + sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ + .subquery() + + # set country code to None if lat or lon changed + upd = app.session.query(Receiver) \ + .filter(and_(Receiver.name == sq.c.name, + or_(Receiver.latitude != sq.c.latitude, + Receiver.longitude != sq.c.longitude) + ) + ) \ + .update({"latitude": sq.c.latitude, + "longitude": sq.c.longitude, + "country_code": None}) + + logger.info("Count of receivers who changed lat or lon: {}".format(upd)) + app.session.commit() + + # update lastseen of known receivers + upd = app.session.query(Receiver) \ + .filter(Receiver.name == sq.c.name) \ + .update({"altitude": sq.c.altitude, + "lastseen": sq.c.lastseen, + "version": sq.c.version, + "platform": sq.c.platform}) + + logger.info("Count of receivers who where updated: {}".format(upd)) + + ## add new receivers + empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, + ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ + .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \ + .filter(Receiver.name == null()) \ + .order_by(ReceiverBeacon.name) + + for receiver_beacon in empty_sq.all(): + receiver = Receiver() + receiver.name = receiver_beacon.name + receiver.latitude = receiver_beacon.latitude + receiver.longitude = receiver_beacon.longitude + receiver.altitude = receiver_beacon.altitude + receiver.firstseen = receiver_beacon.firstseen + receiver.lastseen = receiver_beacon.lastseen + receiver.version = receiver_beacon.version + receiver.platform = receiver_beacon.platform + + app.session.add(receiver) + logger.info("{} added".format(receiver.name)) + + app.session.commit() + + # update country code if None + unknown_country_query = app.session.query(Receiver) \ + .filter(Receiver.country_code == null()) \ + .order_by(Receiver.name) + + for receiver in unknown_country_query.all(): + receiver.country_code = get_country_code(receiver.latitude, receiver.longitude) + logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + + app.session.commit() diff --git a/ogn/commands/showreceiver.py b/ogn/commands/showreceiver.py index ed378d5..0489584 100644 --- a/ogn/commands/showreceiver.py +++ b/ogn/commands/showreceiver.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta -from sqlalchemy.sql import func -from sqlalchemy import distinct, and_ +from sqlalchemy.sql import func, null +from sqlalchemy.sql.functions import coalesce +from sqlalchemy import distinct, and_, or_ from ogn.model import ReceiverBeacon, Receiver from ogn.commands.dbutils import session @@ -13,8 +14,7 @@ receiver_beacons_per_day = 24 * 60 / 5 @manager.command def list_all(): - """Show a list of all receivers (NOT IMPLEMENTED).""" - + """Show a list of all receivers.""" timestamp_24h_ago = datetime.utcnow() - timedelta(days=1) sq = session.query(distinct(ReceiverBeacon.name).label('name'), diff --git a/ogn/utils.py b/ogn/utils.py index a8f546d..fa4a0e5 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -5,6 +5,7 @@ from io import StringIO from .model import Device, AddressOrigin from geopy.geocoders import Nominatim +from geopy.exc import GeopyError DDB_URL = "http://ddb.glidernet.org/download" @@ -54,11 +55,13 @@ def get_trackable(ddb): def get_country_code(latitude, longitude): geolocator = Nominatim() - location = geolocator.reverse("%f, %f" % (latitude, longitude)) try: + location = geolocator.reverse("%f, %f" % (latitude, longitude)) country_code = location.raw["address"]["country_code"] except KeyError: country_code = None + except GeopyError: + country_code = None return country_code @@ -71,7 +74,6 @@ def haversine_distance(location0, location1): lon1 = radians(location1[1]) distance = 6366000 * 2 * asin(sqrt((sin((lat0 - lat1) / 2))**2 + cos(lat0) * cos(lat1) * (sin((lon0 - lon1) / 2))**2)) - print(distance) phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1))) return distance, phi From 03ea96c71eeeab3f70b0c5584e85ee13516ecbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 3 Feb 2016 23:11:54 +0100 Subject: [PATCH 2/8] Added GeocoderTimedOut exception --- tests/test_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c722b67..fa5986b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ import unittest +import unittest.mock as mock from ogn.utils import get_ddb, get_trackable, get_country_code, haversine_distance from ogn.model import AddressOrigin @@ -44,6 +45,15 @@ class TestStringMethods(unittest.TestCase): country_code = get_country_code(latitude, longitude) self.assertEqual(country_code, None) + @mock.patch('ogn.utils.Nominatim') + def test_gec_country_code_exception(self, nominatim_mock): + from geopy.exc import GeocoderTimedOut + instance = nominatim_mock.return_value + + instance.reverse.side_effect = GeocoderTimedOut('Too busy') + country_code = get_country_code(0, 0) + self.assertIsNone(country_code) + def test_haversine_distance(self): # delta: one latitude degree location0 = (0, 0) From 018ea8c549da7fe06aac29d221cbd07237dacfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 3 Feb 2016 23:19:25 +0100 Subject: [PATCH 3/8] Changed obsolete string formatting operations --- ogn/collect/database.py | 4 ++-- ogn/collect/logbook.py | 2 +- ogn/commands/logbook.py | 2 +- ogn/utils.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ogn/collect/database.py b/ogn/collect/database.py index b62ddff..3d193cb 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -30,7 +30,7 @@ def import_ddb(): logger.info("Import registered devices fom the DDB...") counter = update_devices(app.session, AddressOrigin.ogn_ddb, get_ddb()) - logger.info("Imported %i devices." % counter) + logger.info("Imported {} devices.".format(counter)) @app.task @@ -39,7 +39,7 @@ def import_file(path='tests/custom_ddb.txt'): logger.info("Import registered devices from '{}'...".format(path)) counter = update_devices(app.session, AddressOrigin.user_defined, get_ddb(path)) - logger.info("Imported %i devices." % counter) + logger.info("Imported {} devices.".format(counter)) @app.task diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index aa0834d..a748821 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -86,6 +86,6 @@ def compute_takeoff_and_landing(): result = app.session.execute(ins) counter = result.rowcount app.session.commit() - logger.debug("New/recalculated takeoffs and landings: %s" % counter) + logger.debug("New/recalculated takeoffs and landings: {}".format(counter)) return counter diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index 3e76a39..2bda31c 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -21,7 +21,7 @@ def compute(): print("Compute takeoffs and landings...") result = compute_takeoff_and_landing.delay() counter = result.get() - print("New/recalculated takeoffs/landings: %s" % counter) + print("New/recalculated takeoffs/landings: {}"format(counter)) @manager.command diff --git a/ogn/utils.py b/ogn/utils.py index fa4a0e5..c66ca08 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -49,15 +49,15 @@ def get_trackable(ddb): l = [] for i in ddb: if i.tracked and i.address_type in address_prefixes: - l.append('{}{}'.format(address_prefixes[i.address_type], i.address)) + l.append("{}{}".format(address_prefixes[i.address_type], i.address)) return l def get_country_code(latitude, longitude): geolocator = Nominatim() try: - location = geolocator.reverse("%f, %f" % (latitude, longitude)) - country_code = location.raw["address"]["country_code"] + location = geolocator.reverse("{}, {}".format(latitude, longitude)) + country_code = location.raw['address']['country_code'] except KeyError: country_code = None except GeopyError: From 528bcd8e911e1a21c90351ae75e852e45d21414b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Wed, 3 Feb 2016 23:22:04 +0100 Subject: [PATCH 4/8] Removed obsolete string formatting operations --- ogn/commands/logbook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index 2bda31c..aa9a993 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -21,7 +21,7 @@ def compute(): print("Compute takeoffs and landings...") result = compute_takeoff_and_landing.delay() counter = result.get() - print("New/recalculated takeoffs/landings: {}"format(counter)) + print("New/recalculated takeoffs/landings: {}".format(counter)) @manager.command @@ -140,7 +140,7 @@ def show(airport_name, latitude, longitude, altitude): .outerjoin(Device, union_query.c.address == Device.address) \ .order_by(union_query.c.reftime) - print('--- Logbook (%s) ---' % airport_name) + print('--- Logbook ({}) ---'.format(airport_name)) def none_datetime_replacer(datetime_object): return '--:--:--' if datetime_object is None else datetime_object.time() From 0cd46d2be2cf87567049bbbebb1280daf48de874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 4 Feb 2016 18:31:58 +0100 Subject: [PATCH 5/8] Moved receiver update from ogn.collect.database to ogn.collect.receiver --- ogn/collect/celery.py | 4 +- ogn/collect/database.py | 85 +-------------------------------------- ogn/collect/receiver.py | 89 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 84 deletions(-) create mode 100644 ogn/collect/receiver.py diff --git a/ogn/collect/celery.py b/ogn/collect/celery.py index 6ef5479..aa45ad8 100644 --- a/ogn/collect/celery.py +++ b/ogn/collect/celery.py @@ -26,7 +26,9 @@ def close_db(signal, sender): app = Celery('ogn.collect', include=["ogn.collect.database", + "ogn.collect.heatmap", "ogn.collect.logbook", - "ogn.collect.heatmap"]) + "ogn.collect.receiver" + ]) app.config_from_envvar("OGN_CONFIG_MODULE") diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 3d193cb..4697fe5 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -1,11 +1,7 @@ -from sqlalchemy.sql import func, null -from sqlalchemy.sql.functions import coalesce -from sqlalchemy import and_, or_ - from celery.utils.log import get_task_logger -from ogn.model import Device, AddressOrigin, Receiver, ReceiverBeacon -from ogn.utils import get_ddb, get_country_code +from ogn.model import Device, AddressOrigin +from ogn.utils import get_ddb from ogn.collect.celery import app @@ -40,80 +36,3 @@ def import_file(path='tests/custom_ddb.txt'): logger.info("Import registered devices from '{}'...".format(path)) counter = update_devices(app.session, AddressOrigin.user_defined, get_ddb(path)) logger.info("Imported {} devices.".format(counter)) - - -@app.task -def update_receivers(): - """Update the receiver table.""" - ## get current receiver data - last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ - .subquery() - - last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ - .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ - .group_by(ReceiverBeacon.name) \ - .subquery() - - ## update existing receivers - sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ - .subquery() - - # set country code to None if lat or lon changed - upd = app.session.query(Receiver) \ - .filter(and_(Receiver.name == sq.c.name, - or_(Receiver.latitude != sq.c.latitude, - Receiver.longitude != sq.c.longitude) - ) - ) \ - .update({"latitude": sq.c.latitude, - "longitude": sq.c.longitude, - "country_code": None}) - - logger.info("Count of receivers who changed lat or lon: {}".format(upd)) - app.session.commit() - - # update lastseen of known receivers - upd = app.session.query(Receiver) \ - .filter(Receiver.name == sq.c.name) \ - .update({"altitude": sq.c.altitude, - "lastseen": sq.c.lastseen, - "version": sq.c.version, - "platform": sq.c.platform}) - - logger.info("Count of receivers who where updated: {}".format(upd)) - - ## add new receivers - empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, - ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ - .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \ - .filter(Receiver.name == null()) \ - .order_by(ReceiverBeacon.name) - - for receiver_beacon in empty_sq.all(): - receiver = Receiver() - receiver.name = receiver_beacon.name - receiver.latitude = receiver_beacon.latitude - receiver.longitude = receiver_beacon.longitude - receiver.altitude = receiver_beacon.altitude - receiver.firstseen = receiver_beacon.firstseen - receiver.lastseen = receiver_beacon.lastseen - receiver.version = receiver_beacon.version - receiver.platform = receiver_beacon.platform - - app.session.add(receiver) - logger.info("{} added".format(receiver.name)) - - app.session.commit() - - # update country code if None - unknown_country_query = app.session.query(Receiver) \ - .filter(Receiver.country_code == null()) \ - .order_by(Receiver.name) - - for receiver in unknown_country_query.all(): - receiver.country_code = get_country_code(receiver.latitude, receiver.longitude) - logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) - - app.session.commit() diff --git a/ogn/collect/receiver.py b/ogn/collect/receiver.py new file mode 100644 index 0000000..e31a1a5 --- /dev/null +++ b/ogn/collect/receiver.py @@ -0,0 +1,89 @@ +from sqlalchemy.sql import func, null +from sqlalchemy.sql.functions import coalesce +from sqlalchemy import and_, or_ + +from celery.utils.log import get_task_logger + +from ogn.model import Receiver, ReceiverBeacon +from ogn.utils import get_country_code +from ogn.collect.celery import app + +logger = get_task_logger(__name__) + + +@app.task +def update_receivers(): + """Update the receiver table.""" + ## get current receiver data + last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ + .subquery() + + last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ + .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ + .group_by(ReceiverBeacon.name) \ + .subquery() + + ## update existing receivers + sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ + .subquery() + + # set country code to None if lat or lon changed + upd = app.session.query(Receiver) \ + .filter(and_(Receiver.name == sq.c.name, + or_(Receiver.latitude != sq.c.latitude, + Receiver.longitude != sq.c.longitude) + ) + ) \ + .update({"latitude": sq.c.latitude, + "longitude": sq.c.longitude, + "country_code": None}) + + logger.info("Count of receivers who changed lat or lon: {}".format(upd)) + app.session.commit() + + # update lastseen of known receivers + upd = app.session.query(Receiver) \ + .filter(Receiver.name == sq.c.name) \ + .update({"altitude": sq.c.altitude, + "lastseen": sq.c.lastseen, + "version": sq.c.version, + "platform": sq.c.platform}) + + logger.info("Count of receivers who where updated: {}".format(upd)) + + ## add new receivers + empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, + ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ + .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \ + .filter(Receiver.name == null()) \ + .order_by(ReceiverBeacon.name) + + for receiver_beacon in empty_sq.all(): + receiver = Receiver() + receiver.name = receiver_beacon.name + receiver.latitude = receiver_beacon.latitude + receiver.longitude = receiver_beacon.longitude + receiver.altitude = receiver_beacon.altitude + receiver.firstseen = receiver_beacon.firstseen + receiver.lastseen = receiver_beacon.lastseen + receiver.version = receiver_beacon.version + receiver.platform = receiver_beacon.platform + + app.session.add(receiver) + logger.info("{} added".format(receiver.name)) + + app.session.commit() + + # update country code if None + unknown_country_query = app.session.query(Receiver) \ + .filter(Receiver.country_code == null()) \ + .order_by(Receiver.name) + + for receiver in unknown_country_query.all(): + receiver.country_code = get_country_code(receiver.latitude, receiver.longitude) + if receiver.country_code is not None: + logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) + + app.session.commit() From 7543130c2232e0f3aced4b68b901c0085dd1ff74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 4 Feb 2016 18:33:16 +0100 Subject: [PATCH 6/8] Removed (NOT IMPLEMENTED) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 054545b..554df50 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ available commands: [show.receiver] hardware_stats Show some statistics of receiver hardware. - list_all Show a list of all receivers (NOT IMPLEMENTED). + list_all Show a list of all receivers. software_stats Show some statistics of receiver software. ``` @@ -151,7 +151,7 @@ Only the command `logbook.compute` requires a running task server (celery) at th - `import_ddb` - Import registered devices from the ddb - `import_file` - Import registered devices from a local file - ogn.collect.receiver - - `populate` - Generate Receiver table (NOT IMPLEMENTED) + - `update_receivers` - Populate/update receiver table - ogn.collect.logbook - `compute_takeoff_and_landing` - Generate TakeoffLanding table - ogn.collect.heatmap From 23a364761f0ef53472ad52df18421411876aa5c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 4 Feb 2016 19:14:31 +0100 Subject: [PATCH 7/8] fixed flake8 --- ogn/collect/receiver.py | 18 +++++++++--------- ogn/commands/showreceiver.py | 5 ++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ogn/collect/receiver.py b/ogn/collect/receiver.py index e31a1a5..10b10cd 100644 --- a/ogn/collect/receiver.py +++ b/ogn/collect/receiver.py @@ -14,21 +14,21 @@ logger = get_task_logger(__name__) @app.task def update_receivers(): """Update the receiver table.""" - ## get current receiver data + # get current receiver data last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ - .subquery() + .subquery() last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ - .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ - .group_by(ReceiverBeacon.name) \ - .subquery() + .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ + .group_by(ReceiverBeacon.name) \ + .subquery() - ## update existing receivers + # update existing receivers sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ .subquery() - # set country code to None if lat or lon changed + # -set country code to None if lat or lon changed upd = app.session.query(Receiver) \ .filter(and_(Receiver.name == sq.c.name, or_(Receiver.latitude != sq.c.latitude, @@ -42,7 +42,7 @@ def update_receivers(): logger.info("Count of receivers who changed lat or lon: {}".format(upd)) app.session.commit() - # update lastseen of known receivers + # -update lastseen of known receivers upd = app.session.query(Receiver) \ .filter(Receiver.name == sq.c.name) \ .update({"altitude": sq.c.altitude, @@ -52,7 +52,7 @@ def update_receivers(): logger.info("Count of receivers who where updated: {}".format(upd)) - ## add new receivers + # add new receivers empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ diff --git a/ogn/commands/showreceiver.py b/ogn/commands/showreceiver.py index 0489584..1d8c374 100644 --- a/ogn/commands/showreceiver.py +++ b/ogn/commands/showreceiver.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta -from sqlalchemy.sql import func, null -from sqlalchemy.sql.functions import coalesce -from sqlalchemy import distinct, and_, or_ +from sqlalchemy.sql import func +from sqlalchemy import distinct, and_ from ogn.model import ReceiverBeacon, Receiver from ogn.commands.dbutils import session From e85e01e2c101e0c2453437eef2bea441983a4efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 4 Feb 2016 19:18:57 +0100 Subject: [PATCH 8/8] Corrected path --- config/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.py b/config/default.py index 32ef2b1..6a0e3c8 100644 --- a/config/default.py +++ b/config/default.py @@ -12,7 +12,7 @@ CELERYBEAT_SCHEDULE = { 'schedule': timedelta(minutes=5), }, 'update-receiver-table': { - 'task': 'ogn.collect.database.update_receivers', + 'task': 'ogn.collect.receiver.update_receivers', 'schedule': timedelta(minutes=5), }, }