Merge pull request #35 from kerel-fs/feature/receiverdistance

Add central configuration & spherical calculations & other stuff

README.md

@@ -81,18 +81,33 @@ To schedule tasks like takeoff/landing-detection (`logbook.compute`),
 The following scripts run in the foreground and should be deamonized
 (eg. use [supervisord](http://supervisord.org/)).
 
-- start aprs client
+- Start the aprs client
 
   ```
   ./manage.py gateway.run
   ```
 
-- start task server (make sure redis is up and running)
+- Start a task server (make sure redis is up and running)
 
   ```
   celery -A ogn.collect worker -l info
   ```
 
+- Start the task scheduler (make sure a task server is up and running)
+
+  ```
+  celery -A ogn.collect beat -l info
+  ```
+
+
+To load a custom configuration, create a file `myconfig.py` (see [config/default.py](config/default.py))
+and set the environment variable `OGN_CONFIG_MODULE` accordingly.
+
+```
+export OGN_CONFIG_MODULE="myconfig.py"
+./manage.py gateway.run
+```
+
 ### manage.py - CLI options
 ```
 usage: manage.py [<namespace>.]<command> [<args>]
@@ -130,14 +145,17 @@ available commands:
 Only the command `logbook.compute` requires a running task server (celery) at the moment.
 
 
-### Scheduled tasks
+### Available tasks
 - ogn.collect.database
   - `import_ddb` - Import registered devices from the ddb
   - `import_file` - Import registered devices from a local file
 - ogn.collect.receiver
-  - `populate` - generate Receiver table (NOT IMPLEMENTED)
+  - `populate` - Generate Receiver table (NOT IMPLEMENTED)
 - ogn.collect.logbook
-  - `compute_takeoff_and_landing` - generate TakeoffLanding table
+  - `compute_takeoff_and_landing` - Generate TakeoffLanding table
+- ogn.collect.heatmap
+  - `update_beacon_receiver_distance_all` - Calculate the distance between aircraft and
+    receiver for the last aircraft beacons
 
 
 ## License

config/default.py

@@ -0,0 +1,16 @@
+SQLALCHEMY_DATABASE_URI = 'sqlite:///beacons.db'
+
+BROKER_URL = 'redis://localhost:6379/0'
+CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
+
+
+from datetime import timedelta
+
+CELERYBEAT_SCHEDULE = {
+    'update-receiver-distance': {
+        'task': 'ogn.collect.heatmap.update_beacon_receiver_distance_all',
+        'schedule': timedelta(minutes=5),
+    },
+}
+
+CELERY_TIMEZONE = 'UTC'

ogn/collect/celery.py

@@ -1,21 +1,19 @@
+import os
+import importlib
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from celery import Celery
 from celery.signals import worker_init, worker_shutdown
 
-app = Celery('ogn.collect',
-             broker='redis://localhost:6379/0',
-             backend='redis://localhost:6379/0',
-             include=["ogn.collect.database", "ogn.collect.logbook"])
-
-DB_URI = 'sqlite:///beacons.db'
+os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default')
+config = importlib.import_module(os.environ['OGN_CONFIG_MODULE'])
 
 
 @worker_init.connect
 def connect_db(signal, sender):
     # Load settings like DB_URI...
-    engine = create_engine(DB_URI, echo=False)
+    engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False)
 
     Session = sessionmaker(bind=engine)
     sender.app.session = Session()
@@ -26,5 +24,9 @@ def close_db(signal, sender):
     sender.app.session.close()
 
 
-if __name__ == '__main__':
-    app.start()
+app = Celery('ogn.collect',
+             include=["ogn.collect.database",
+                      "ogn.collect.logbook",
+                      "ogn.collect.heatmap"])
+
+app.config_from_envvar("OGN_CONFIG_MODULE")

ogn/commands/database.py

@@ -1,6 +1,3 @@
-from alembic.config import Config
-from alembic import command
-
 from ogn.commands.dbutils import engine, session
 from ogn.model import Base, AddressOrigin
 from ogn.utils import get_ddb
@@ -14,6 +11,9 @@ manager = Manager()
 def init():
     """Initialize the database."""
 
+    from alembic.config import Config
+    from alembic import command
+
     Base.metadata.create_all(engine)
     alembic_cfg = Config("alembic.ini")
     command.stamp(alembic_cfg, "head")

ogn/commands/dbutils.py

@@ -1,6 +1,14 @@
+import os
+import importlib
+
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-engine = create_engine('sqlite:///beacons.db', echo=False)
+
+
+os.environ.setdefault('OGN_CONFIG_MODULE', 'config.default')
+
+config = importlib.import_module(os.environ['OGN_CONFIG_MODULE'])
+engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False)
 
 Session = sessionmaker(bind=engine)
 session = Session()

ogn/gateway/client.py

@@ -84,7 +84,7 @@ class ognGateway:
             self.logger.error('OgnParseError while parsing line: {}'.format(line), exc_info=True)
             return
         except AmbigousTimeError as e:
-            self.logger.error('Drop packet, {:.0f}s from past.'.format(e.timedelta.total_seconds()))
+            self.logger.error('Drop packet, {:.0f}s from past: {}'.format(e.timedelta.total_seconds(), line))
             return
 
         if beacon is not None:

ogn/gateway/manage.py

@@ -4,6 +4,7 @@ from ogn.gateway.client import ognGateway
 from ogn.commands.dbutils import session
 
 from manager import Manager
+
 manager = Manager()
 
 logging_formatstr = '%(asctime)s - %(levelname).4s - %(name)s - %(message)s'

ogn/utils.py

@@ -60,3 +60,18 @@ def get_country_code(latitude, longitude):
     except KeyError:
         country_code = None
     return country_code
+
+
+def haversine_distance(location0, location1):
+    from math import asin, sqrt, sin, cos, atan2, radians, degrees
+
+    lat0 = radians(location0[0])
+    lon0 = radians(location0[1])
+    lat1 = radians(location1[0])
+    lon1 = radians(location1[1])
+
+    distance = 6366000 * 2 * asin(sqrt((sin((lat0 - lat1) / 2))**2 + cos(lat0) * cos(lat1) * (sin((lon0 - lon1) / 2))**2))
+    print(distance)
+    phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1)))
+
+    return distance, phi

setup.py

@@ -51,6 +51,9 @@ setup(
             'coveralls==0.4.4',
             'flake8==2.5.0'
         ],
+        'postgresql': [
+            'psycopg2==2.6.1'
+        ]
     },
     zip_safe=False
 )

tests/test_utils.py

@@ -1,6 +1,6 @@
 import unittest
 
-from ogn.utils import get_ddb, get_trackable, get_country_code
+from ogn.utils import get_ddb, get_trackable, get_country_code, haversine_distance
 from ogn.model import AddressOrigin
 
 
@@ -43,3 +43,28 @@ class TestStringMethods(unittest.TestCase):
         longitude = -0.0009119
         country_code = get_country_code(latitude, longitude)
         self.assertEqual(country_code, None)
+
+    def test_haversine_distance(self):
+        # delta: one latitude degree
+        location0 = (0, 0)
+        location1 = (-1, 0)
+
+        (distance, phi) = haversine_distance(location0, location1)
+        self.assertAlmostEqual(distance, 60 * 1852, -2)
+        self.assertEqual(phi, 180)
+
+        # delta: one longitude degree at the equator
+        location0 = (0, 0)
+        location1 = (0, -1)
+
+        (distance, phi) = haversine_distance(location0, location1)
+        self.assertAlmostEqual(distance, 60 * 1852, -2)
+        self.assertEqual(phi, 90)
+
+        # delta: 29000m
+        location0 = (48.865, 9.2225)
+        location1 = (48.74435, 9.578)
+
+        (distance, phi) = haversine_distance(location0, location1)
+        self.assertAlmostEqual(distance, 29265.6035812215, -1)
+        self.assertAlmostEqual(phi, -117.1275408121, 5)