From 6aec9ee35d94dec070c0de5b3d920c393719eab7 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 02:34:12 +0100 Subject: [PATCH 1/8] Add central configuration by python modules Configuration module is loaded from environment variable OGN_CONFIG_MODULE, defaults to "config/defaults.py". --- README.md | 8 ++++++++ config/default.py | 4 ++++ ogn/collect/celery.py | 19 ++++++++++--------- ogn/commands/dbutils.py | 9 ++++++++- 4 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 config/default.py diff --git a/README.md b/README.md index 4542982..87b6e3d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,14 @@ The following scripts run in the foreground and should be deamonized celery -A ogn.collect worker -l info ``` +To load a custom configuration, create a file `myconfig.py` (see [config/default.py](config/default.py)) +and set the environment variable `OGN_CONFIG_MODULE` accordingly. + +``` +export OGN_CONFIG_MODULE="myconfig.py" +./manage.py gateway.run +``` + ### manage.py - CLI options ``` usage: manage.py [.] [] diff --git a/config/default.py b/config/default.py new file mode 100644 index 0000000..d9ce11a --- /dev/null +++ b/config/default.py @@ -0,0 +1,4 @@ +SQLALCHEMY_DATABASE_URI = 'sqlite:///beacons.db' + +BROKER_URL = 'redis://localhost:6379/0' +CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' diff --git a/ogn/collect/celery.py b/ogn/collect/celery.py index 3fbb2b9..816e90b 100644 --- a/ogn/collect/celery.py +++ b/ogn/collect/celery.py @@ -1,21 +1,19 @@ +import os +import importlib from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from celery import Celery from celery.signals import worker_init, worker_shutdown -app = Celery('ogn.collect', - broker='redis://localhost:6379/0', - backend='redis://localhost:6379/0', - include=["ogn.collect.database", "ogn.collect.logbook"]) - -DB_URI = 'sqlite:///beacons.db' +os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default') +config = importlib.import_module(os.environ['OGN_CONFIG_MODULE']) @worker_init.connect def connect_db(signal, sender): # Load settings like DB_URI... - engine = create_engine(DB_URI, echo=False) + engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False) Session = sessionmaker(bind=engine) sender.app.session = Session() @@ -26,5 +24,8 @@ def close_db(signal, sender): sender.app.session.close() -if __name__ == '__main__': - app.start() +app = Celery('ogn.collect', + include=["ogn.collect.database", + "ogn.collect.logbook"]) + +app.config_from_envvar("OGN_CONFIG_MODULE") diff --git a/ogn/commands/dbutils.py b/ogn/commands/dbutils.py index 07ca652..be2d294 100644 --- a/ogn/commands/dbutils.py +++ b/ogn/commands/dbutils.py @@ -1,6 +1,13 @@ +import os +import importlib from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -engine = create_engine('sqlite:///beacons.db', echo=False) + + +os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default') + +config = importlib.import_module(os.environ['OGN_CONFIG_MODULE']) +engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False) Session = sessionmaker(bind=engine) session = Session() From 794ab5748f2dfd9cd9986451638e0ba7dbcb03e2 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 02:38:55 +0100 Subject: [PATCH 2/8] CLI: Move alembic imports inside the calling function Reduces execution time of "manage.py" a bit. An import of alembic.config takes a relatively long time and "manage.py" by itself is slow. --- ogn/commands/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 0f7d0fc..c534044 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -1,6 +1,3 @@ -from alembic.config import Config -from alembic import command - from ogn.commands.dbutils import engine, session from ogn.model import Base, AddressOrigin from ogn.utils import get_ddb @@ -14,6 +11,9 @@ manager = Manager() def init(): """Initialize the database.""" + from alembic.config import Config + from alembic import command + Base.metadata.create_all(engine) alembic_cfg = Config("alembic.ini") command.stamp(alembic_cfg, "head") From e22e6a619e00421932cc09d7b61c2e87c42c39d4 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 02:49:58 +0100 Subject: [PATCH 3/8] gateway: Add logging of dropped packets --- ogn/gateway/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ogn/gateway/client.py b/ogn/gateway/client.py index 53e359f..cb8d806 100644 --- a/ogn/gateway/client.py +++ b/ogn/gateway/client.py @@ -84,7 +84,7 @@ class ognGateway: self.logger.error('OgnParseError while parsing line: {}'.format(line), exc_info=True) return except AmbigousTimeError as e: - self.logger.error('Drop packet, {:.0f}s from past.'.format(e.timedelta.total_seconds())) + self.logger.error('Drop packet, {:.0f}s from past: {}'.format(e.timedelta.total_seconds(), line)) return if beacon is not None: From 88406a31d00b43d30b02565945aa376ba1cfce90 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 02:54:51 +0100 Subject: [PATCH 4/8] setup.py: Add optional postresql requirement To install ogn-python with postresql support (currently required for logbook.compute/.show), use ``` pip install -e .[postgresql] ``` --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 3fe39f3..f17591b 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,9 @@ setup( 'coveralls==0.4.4', 'flake8==2.5.0' ], + 'postgresql': [ + 'psycopg2==2.6.1' + ] }, zip_safe=False ) From 6cc6378496188b1e4abba033af9043cffa626f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Thu, 28 Jan 2016 20:29:08 +0100 Subject: [PATCH 5/8] Add (slow) [radius, theta, phi] calculation --- ogn/commands/dbutils.py | 30 ++++++++++++++++++++++- ogn/gateway/manage.py | 6 ++++- ogn/utils.py | 21 ++++++++++++++++ tests/test_utils.py | 53 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/ogn/commands/dbutils.py b/ogn/commands/dbutils.py index be2d294..070d7fc 100644 --- a/ogn/commands/dbutils.py +++ b/ogn/commands/dbutils.py @@ -1,8 +1,13 @@ import os import importlib -from sqlalchemy import create_engine + +from sqlalchemy import create_engine, and_, desc +from sqlalchemy.sql import null from sqlalchemy.orm import sessionmaker +from ogn.model import AircraftBeacon, ReceiverBeacon +from ogn.utils import wgs84_to_sphere + os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default') @@ -11,3 +16,26 @@ engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False) Session = sessionmaker(bind=engine) session = Session() + + +def update_receiver_childs(name): + last_receiver_beacon = session.query(ReceiverBeacon) \ + .filter(ReceiverBeacon.name == name) \ + .order_by(desc(ReceiverBeacon.timestamp)) \ + .first() + + if (last_receiver_beacon is None): + return + + aircraft_beacons_query = session.query(AircraftBeacon) \ + .filter(and_(AircraftBeacon.timestamp > last_receiver_beacon.timestamp, + AircraftBeacon.receiver_name == name, + AircraftBeacon.radius == null())) + + for aircraft_beacon in aircraft_beacons_query.all(): + [radius, theta, phi] = wgs84_to_sphere(last_receiver_beacon, + aircraft_beacon) + aircraft_beacon.radius = radius + aircraft_beacon.theta = theta + aircraft_beacon.phi = phi + session.commit() diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index d7ce903..063cd17 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -1,9 +1,11 @@ import logging from ogn.gateway.client import ognGateway -from ogn.commands.dbutils import session +from ogn.commands.dbutils import session, update_receiver_childs +from ogn.model import ReceiverBeacon from manager import Manager + manager = Manager() logging_formatstr = '%(asctime)s - %(levelname).4s - %(name)s - %(message)s' @@ -33,6 +35,8 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): gateway.connect() def process_beacon(beacon): + if isinstance(beacon, ReceiverBeacon): + update_receiver_childs(beacon.name) session.add(beacon) session.commit() diff --git a/ogn/utils.py b/ogn/utils.py index e1219aa..698ef68 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -60,3 +60,24 @@ def get_country_code(latitude, longitude): except KeyError: country_code = None return country_code + + +def wgs84_to_sphere(receiver_beacon, aircraft_beacon): + from math import pi, asin, sqrt, sin, cos, atan2 + deg2rad = pi / 180 + rad2deg = 180 / pi + + lat1 = receiver_beacon.latitude * deg2rad + lon1 = receiver_beacon.longitude * deg2rad + alt1 = receiver_beacon.altitude + + lat2 = aircraft_beacon.latitude * deg2rad + lon2 = aircraft_beacon.longitude * deg2rad + alt2 = aircraft_beacon.altitude + + distance = 6366000 * 2 * asin(sqrt((sin((lat1 - lat2) / 2))**2 + cos(lat1) * cos(lat2) * (sin((lon1 - lon2) / 2))**2)) + theta = atan2(alt2 - alt1, distance) * rad2deg + phi = atan2(sin(lon1 - lon2) * cos(lat2), cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(lon1 - lon2)) * rad2deg + + radius = sqrt(distance**2 + (alt2 - alt1)**2) + return radius, theta, phi diff --git a/tests/test_utils.py b/tests/test_utils.py index 8b2d03c..8a18173 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import unittest -from ogn.utils import get_ddb, get_trackable, get_country_code -from ogn.model import AddressOrigin +from ogn.utils import get_ddb, get_trackable, get_country_code, wgs84_to_sphere +from ogn.model import AddressOrigin, Beacon class TestStringMethods(unittest.TestCase): @@ -43,3 +43,52 @@ class TestStringMethods(unittest.TestCase): longitude = -0.0009119 country_code = get_country_code(latitude, longitude) self.assertEqual(country_code, None) + + def test_wgs84_to_sphere(self): + receiver_beacon = Beacon() + receiver_beacon.latitude = 0 + receiver_beacon.longitude = 0 + receiver_beacon.altitude = 0 + + # delta: one latitude degree + aircraft_beacon = Beacon() + aircraft_beacon.latitude = -1 + aircraft_beacon.longitude = 0 + aircraft_beacon.altitude = 0 + [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) + self.assertAlmostEqual(radius, 60 * 1852, -2) + self.assertEqual(theta, 0) + self.assertEqual(phi, 180) + + # delta: one longitude degree at the equator + aircraft_beacon.latitude = 0 + aircraft_beacon.longitude = -1 + aircraft_beacon.altitude = 0 + [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) + self.assertAlmostEqual(radius, 60 * 1852, -2) + self.assertEqual(theta, 0) + self.assertEqual(phi, 90) + + # delta: 1000m altitude + aircraft_beacon.latitude = 0 + aircraft_beacon.longitude = 0 + aircraft_beacon.altitude = 1000 + [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) + self.assertAlmostEqual(radius, 1000, 3) + self.assertEqual(theta, 90) + self.assertEqual(phi, 0) + + # receiver + receiver_beacon.latitude = 48.865 + receiver_beacon.longitude = 9.2225 + receiver_beacon.altitude = 574 + + # aircraft beacon + aircraft_beacon.latitude = 48.74435 + aircraft_beacon.longitude = 9.578 + aircraft_beacon.altitude = 929 + + [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) + self.assertAlmostEqual(radius, 29265.6035812215, -1) + self.assertAlmostEqual(theta, 0.694979846308314, 5) + self.assertAlmostEqual(phi, -117.1275408121, 5) From d3c38924e23305e0342e0c37595bf191fcfe5915 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 05:22:07 +0100 Subject: [PATCH 6/8] ogn/utils.py: Rename wgs84_to_sphere to haversine_distance --- ogn/commands/dbutils.py | 15 ++++++++--- ogn/utils.py | 26 +++++++----------- tests/test_utils.py | 60 +++++++++++++---------------------------- 3 files changed, 39 insertions(+), 62 deletions(-) diff --git a/ogn/commands/dbutils.py b/ogn/commands/dbutils.py index 070d7fc..99bfaf3 100644 --- a/ogn/commands/dbutils.py +++ b/ogn/commands/dbutils.py @@ -6,7 +6,7 @@ from sqlalchemy.sql import null from sqlalchemy.orm import sessionmaker from ogn.model import AircraftBeacon, ReceiverBeacon -from ogn.utils import wgs84_to_sphere +from ogn.utils import haversine_distance os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default') @@ -33,9 +33,16 @@ def update_receiver_childs(name): AircraftBeacon.radius == null())) for aircraft_beacon in aircraft_beacons_query.all(): - [radius, theta, phi] = wgs84_to_sphere(last_receiver_beacon, - aircraft_beacon) - aircraft_beacon.radius = radius + location0 = (last_receiver_beacon.latitude, last_receiver_beacon.longitude) + location1 = (aircraft_beacon.latitude, aircraft_beacon.longitude) + alt0 = last_receiver_beacon.altitude + alt1 = aircraft_beacon.altitude + + (flat_distance, phi) = haversine_distance(location0, location1) + theta = atan2(alt1 - alt0, flat_distance) * 180 / pi + distance = sqrt(flat_distance**2 + (alt1 - alt0)**2) + + aircraft_beacon.radius = distance aircraft_beacon.theta = theta aircraft_beacon.phi = phi session.commit() diff --git a/ogn/utils.py b/ogn/utils.py index 698ef68..a8f546d 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -62,22 +62,16 @@ def get_country_code(latitude, longitude): return country_code -def wgs84_to_sphere(receiver_beacon, aircraft_beacon): - from math import pi, asin, sqrt, sin, cos, atan2 - deg2rad = pi / 180 - rad2deg = 180 / pi +def haversine_distance(location0, location1): + from math import asin, sqrt, sin, cos, atan2, radians, degrees - lat1 = receiver_beacon.latitude * deg2rad - lon1 = receiver_beacon.longitude * deg2rad - alt1 = receiver_beacon.altitude + lat0 = radians(location0[0]) + lon0 = radians(location0[1]) + lat1 = radians(location1[0]) + lon1 = radians(location1[1]) - lat2 = aircraft_beacon.latitude * deg2rad - lon2 = aircraft_beacon.longitude * deg2rad - alt2 = aircraft_beacon.altitude + distance = 6366000 * 2 * asin(sqrt((sin((lat0 - lat1) / 2))**2 + cos(lat0) * cos(lat1) * (sin((lon0 - lon1) / 2))**2)) + print(distance) + phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1))) - distance = 6366000 * 2 * asin(sqrt((sin((lat1 - lat2) / 2))**2 + cos(lat1) * cos(lat2) * (sin((lon1 - lon2) / 2))**2)) - theta = atan2(alt2 - alt1, distance) * rad2deg - phi = atan2(sin(lon1 - lon2) * cos(lat2), cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(lon1 - lon2)) * rad2deg - - radius = sqrt(distance**2 + (alt2 - alt1)**2) - return radius, theta, phi + return distance, phi diff --git a/tests/test_utils.py b/tests/test_utils.py index 8a18173..c722b67 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import unittest -from ogn.utils import get_ddb, get_trackable, get_country_code, wgs84_to_sphere -from ogn.model import AddressOrigin, Beacon +from ogn.utils import get_ddb, get_trackable, get_country_code, haversine_distance +from ogn.model import AddressOrigin class TestStringMethods(unittest.TestCase): @@ -44,51 +44,27 @@ class TestStringMethods(unittest.TestCase): country_code = get_country_code(latitude, longitude) self.assertEqual(country_code, None) - def test_wgs84_to_sphere(self): - receiver_beacon = Beacon() - receiver_beacon.latitude = 0 - receiver_beacon.longitude = 0 - receiver_beacon.altitude = 0 - + def test_haversine_distance(self): # delta: one latitude degree - aircraft_beacon = Beacon() - aircraft_beacon.latitude = -1 - aircraft_beacon.longitude = 0 - aircraft_beacon.altitude = 0 - [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) - self.assertAlmostEqual(radius, 60 * 1852, -2) - self.assertEqual(theta, 0) + location0 = (0, 0) + location1 = (-1, 0) + + (distance, phi) = haversine_distance(location0, location1) + self.assertAlmostEqual(distance, 60 * 1852, -2) self.assertEqual(phi, 180) # delta: one longitude degree at the equator - aircraft_beacon.latitude = 0 - aircraft_beacon.longitude = -1 - aircraft_beacon.altitude = 0 - [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) - self.assertAlmostEqual(radius, 60 * 1852, -2) - self.assertEqual(theta, 0) + location0 = (0, 0) + location1 = (0, -1) + + (distance, phi) = haversine_distance(location0, location1) + self.assertAlmostEqual(distance, 60 * 1852, -2) self.assertEqual(phi, 90) - # delta: 1000m altitude - aircraft_beacon.latitude = 0 - aircraft_beacon.longitude = 0 - aircraft_beacon.altitude = 1000 - [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) - self.assertAlmostEqual(radius, 1000, 3) - self.assertEqual(theta, 90) - self.assertEqual(phi, 0) + # delta: 29000m + location0 = (48.865, 9.2225) + location1 = (48.74435, 9.578) - # receiver - receiver_beacon.latitude = 48.865 - receiver_beacon.longitude = 9.2225 - receiver_beacon.altitude = 574 - - # aircraft beacon - aircraft_beacon.latitude = 48.74435 - aircraft_beacon.longitude = 9.578 - aircraft_beacon.altitude = 929 - - [radius, theta, phi] = wgs84_to_sphere(receiver_beacon, aircraft_beacon) - self.assertAlmostEqual(radius, 29265.6035812215, -1) - self.assertAlmostEqual(theta, 0.694979846308314, 5) + (distance, phi) = haversine_distance(location0, location1) + self.assertAlmostEqual(distance, 29265.6035812215, -1) self.assertAlmostEqual(phi, -117.1275408121, 5) From ae3788764d78c6f71a259a31070be8d19519014f Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 05:25:35 +0100 Subject: [PATCH 7/8] celery: Add task update_beacon_receiver_distance Replaces ogn.commands.dbutils.update_receiver_childs --- README.md | 7 +++++-- ogn/collect/celery.py | 3 ++- ogn/commands/dbutils.py | 36 +----------------------------------- ogn/gateway/manage.py | 6 +++--- 4 files changed, 11 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 87b6e3d..7248912 100644 --- a/README.md +++ b/README.md @@ -143,9 +143,12 @@ Only the command `logbook.compute` requires a running task server (celery) at th - `import_ddb` - Import registered devices from the ddb - `import_file` - Import registered devices from a local file - ogn.collect.receiver - - `populate` - generate Receiver table (NOT IMPLEMENTED) + - `populate` - Generate Receiver table (NOT IMPLEMENTED) - ogn.collect.logbook - - `compute_takeoff_and_landing` - generate TakeoffLanding table + - `compute_takeoff_and_landing` - Generate TakeoffLanding table +- ogn.collect.heatmap + - `update_beacon_receiver_distance_all` - Calculate the distance between aircraft and + receiver for the last aircraft beacons ## License diff --git a/ogn/collect/celery.py b/ogn/collect/celery.py index 816e90b..6ef5479 100644 --- a/ogn/collect/celery.py +++ b/ogn/collect/celery.py @@ -26,6 +26,7 @@ def close_db(signal, sender): app = Celery('ogn.collect', include=["ogn.collect.database", - "ogn.collect.logbook"]) + "ogn.collect.logbook", + "ogn.collect.heatmap"]) app.config_from_envvar("OGN_CONFIG_MODULE") diff --git a/ogn/commands/dbutils.py b/ogn/commands/dbutils.py index 99bfaf3..3e4a3c4 100644 --- a/ogn/commands/dbutils.py +++ b/ogn/commands/dbutils.py @@ -1,13 +1,9 @@ import os import importlib -from sqlalchemy import create_engine, and_, desc -from sqlalchemy.sql import null +from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from ogn.model import AircraftBeacon, ReceiverBeacon -from ogn.utils import haversine_distance - os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default') @@ -16,33 +12,3 @@ engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False) Session = sessionmaker(bind=engine) session = Session() - - -def update_receiver_childs(name): - last_receiver_beacon = session.query(ReceiverBeacon) \ - .filter(ReceiverBeacon.name == name) \ - .order_by(desc(ReceiverBeacon.timestamp)) \ - .first() - - if (last_receiver_beacon is None): - return - - aircraft_beacons_query = session.query(AircraftBeacon) \ - .filter(and_(AircraftBeacon.timestamp > last_receiver_beacon.timestamp, - AircraftBeacon.receiver_name == name, - AircraftBeacon.radius == null())) - - for aircraft_beacon in aircraft_beacons_query.all(): - location0 = (last_receiver_beacon.latitude, last_receiver_beacon.longitude) - location1 = (aircraft_beacon.latitude, aircraft_beacon.longitude) - alt0 = last_receiver_beacon.altitude - alt1 = aircraft_beacon.altitude - - (flat_distance, phi) = haversine_distance(location0, location1) - theta = atan2(alt1 - alt0, flat_distance) * 180 / pi - distance = sqrt(flat_distance**2 + (alt1 - alt0)**2) - - aircraft_beacon.radius = distance - aircraft_beacon.theta = theta - aircraft_beacon.phi = phi - session.commit() diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index 063cd17..7820edf 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -1,8 +1,8 @@ import logging from ogn.gateway.client import ognGateway -from ogn.commands.dbutils import session, update_receiver_childs -from ogn.model import ReceiverBeacon +from ogn.commands.dbutils import session +from ogn.collect.heatmap import update_beacon_receiver_distance from manager import Manager @@ -36,7 +36,7 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): def process_beacon(beacon): if isinstance(beacon, ReceiverBeacon): - update_receiver_childs(beacon.name) + update_beacon_receiver_distance(beacon.name) session.add(beacon) session.commit() From 31b2aadddab9569436ce47cd01ffa8a520f28acb Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Fri, 29 Jan 2016 05:33:59 +0100 Subject: [PATCH 8/8] celery: Add scheduled task update_beacon_receiver_distance Celery beat is used to call update_beacon_receiver_distance_all every five minutes. The time interval is configurable by adjusting the configuration file. --- README.md | 13 ++++++++++--- config/default.py | 12 ++++++++++++ ogn/gateway/manage.py | 3 --- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7248912..c302bbb 100644 --- a/README.md +++ b/README.md @@ -81,18 +81,25 @@ To schedule tasks like takeoff/landing-detection (`logbook.compute`), The following scripts run in the foreground and should be deamonized (eg. use [supervisord](http://supervisord.org/)). -- start aprs client +- Start the aprs client ``` ./manage.py gateway.run ``` -- start task server (make sure redis is up and running) +- Start a task server (make sure redis is up and running) ``` celery -A ogn.collect worker -l info ``` +- Start the task scheduler (make sure a task server is up and running) + + ``` + celery -A ogn.collect beat -l info + ``` + + To load a custom configuration, create a file `myconfig.py` (see [config/default.py](config/default.py)) and set the environment variable `OGN_CONFIG_MODULE` accordingly. @@ -138,7 +145,7 @@ available commands: Only the command `logbook.compute` requires a running task server (celery) at the moment. -### Scheduled tasks +### Available tasks - ogn.collect.database - `import_ddb` - Import registered devices from the ddb - `import_file` - Import registered devices from a local file diff --git a/config/default.py b/config/default.py index d9ce11a..f78c931 100644 --- a/config/default.py +++ b/config/default.py @@ -2,3 +2,15 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:///beacons.db' BROKER_URL = 'redis://localhost:6379/0' CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' + + +from datetime import timedelta + +CELERYBEAT_SCHEDULE = { + 'update-receiver-distance': { + 'task': 'ogn.collect.heatmap.update_beacon_receiver_distance_all', + 'schedule': timedelta(minutes=5), + }, +} + +CELERY_TIMEZONE = 'UTC' diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index 7820edf..b4804da 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -2,7 +2,6 @@ import logging from ogn.gateway.client import ognGateway from ogn.commands.dbutils import session -from ogn.collect.heatmap import update_beacon_receiver_distance from manager import Manager @@ -35,8 +34,6 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): gateway.connect() def process_beacon(beacon): - if isinstance(beacon, ReceiverBeacon): - update_beacon_receiver_distance(beacon.name) session.add(beacon) session.commit()