Merge pull request #39 from kerel-fs/format/update_receivers

Add note about special backend requirements

README.md

@@ -104,7 +104,8 @@ To load a custom configuration, create a file `myconfig.py` (see [config/default
 and set the environment variable `OGN_CONFIG_MODULE` accordingly.
 
 ```
-export OGN_CONFIG_MODULE="myconfig.py"
+touch myconfig.py
+export OGN_CONFIG_MODULE="myconfig"
 ./manage.py gateway.run
 ```
 
@@ -147,17 +148,20 @@ Only the command `logbook.compute` requires a running task server (celery) at th
 
 
 ### Available tasks
-- ogn.collect.database
-  - `import_ddb` - Import registered devices from the ddb
-  - `import_file` - Import registered devices from a local file
-- ogn.collect.receiver
-  - `update_receivers` - Populate/update receiver table
-- ogn.collect.logbook
-  - `compute_takeoff_and_landing` - Generate TakeoffLanding table
-- ogn.collect.heatmap
-  - `update_beacon_receiver_distance_all` - Calculate the distance between aircraft and
-    receiver for the last aircraft beacons
 
+- `ogn.collect.database.import_ddb` - Import registered devices from the ddb
+- `ogn.collect.database.import_file` - Import registered devices from a local file
+- `ogn.collect.heatmap.update_beacon_receiver_distance_all` - Calculate the distance between aircraft and receiver for the last aircraft beacons
+- `ogn.collect.receiver.update_receivers` - Populate/update receiver table (requires postgresql-backend)
+- `ogn.collect.logbook.compute_takeoff_and_landing` - Generate TakeoffLanding table (requires postgresql-backend)
+
+If the task server is up and running, tasks could be started manually.
+
+```
+python3
+>>>from ogn.collect.database import import_ddb
+>>>import_ddb.delay()
+```
 
 ## License
 Licensed under the [AGPLv3](LICENSE).

config/default.py

@@ -7,14 +7,23 @@ CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
 from datetime import timedelta
 
 CELERYBEAT_SCHEDULE = {
+    'update-ddb': {
+        'task': 'ogn.collect.database.import_ddb',
+        'schedule': timedelta(minutes=15),
+    },
     'update-receiver-distance': {
         'task': 'ogn.collect.heatmap.update_beacon_receiver_distance_all',
         'schedule': timedelta(minutes=5),
     },
-    'update-receiver-table': {
+# Only supported with postgresql backend
-        'task': 'ogn.collect.receiver.update_receivers',
+#    'update-logbook': {
-        'schedule': timedelta(minutes=5),
+#        'task': 'ogn.collect.logbook.compute_takeoff_and_landing',
-    },
+#        'schedule': timedelta(minutes=15),
+#    },
+#    'update-receiver-table': {
+#        'task': 'ogn.collect.receiver.update_receivers',
+#        'schedule': timedelta(minutes=5),
+#    },
 }
 
 CELERY_TIMEZONE = 'UTC'

ogn/collect/receiver.py

@@ -18,44 +18,58 @@ def update_receivers():
     last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \
                                .subquery()
 
-    last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \
+    last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name,
-                                 .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \
+                                                func.min(ReceiverBeacon.timestamp).label('firstseen'),
-                                 .group_by(ReceiverBeacon.name) \
+                                                func.max(ReceiverBeacon.timestamp).label('lastseen')) \
-                                 .subquery()
+                                         .filter(ReceiverBeacon.timestamp >= last_entry_sq.columns.last_entry) \
+                                         .group_by(ReceiverBeacon.name) \
+                                         .subquery()
 
     # update existing receivers
-    sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \
+    sq = app.session.query(ReceiverBeacon.name,
-            .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \
+                           ReceiverBeacon.latitude,
-            .subquery()
+                           ReceiverBeacon.longitude,
+                           ReceiverBeacon.altitude,
+                           last_receiver_beacon_sq.columns.firstseen,
+                           last_receiver_beacon_sq.columns.lastseen,
+                           ReceiverBeacon.version,
+                           ReceiverBeacon.platform) \
+                    .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
+                                 ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
+                    .subquery()
 
-    # -set country code to None if lat or lon changed
+    # set country code to None if lat or lon changed
     upd = app.session.query(Receiver) \
-                     .filter(and_(Receiver.name == sq.c.name,
+                     .filter(and_(Receiver.name == sq.columns.name,
-                                  or_(Receiver.latitude != sq.c.latitude,
+                                  or_(Receiver.latitude != sq.columns.latitude,
-                                      Receiver.longitude != sq.c.longitude)
+                                      Receiver.longitude != sq.columns.longitude))) \
-                                  )
+                     .update({"latitude": sq.columns.latitude,
-                             ) \
+                              "longitude": sq.columns.longitude,
-                     .update({"latitude": sq.c.latitude,
+                              "country_code": ""})
-                              "longitude": sq.c.longitude,
-                              "country_code": None})
 
     logger.info("Count of receivers who changed lat or lon: {}".format(upd))
     app.session.commit()
 
-    # -update lastseen of known receivers
+    # update lastseen of known receivers
     upd = app.session.query(Receiver) \
-                     .filter(Receiver.name == sq.c.name) \
+                     .filter(Receiver.name == sq.columns.name) \
-                     .update({"altitude": sq.c.altitude,
+                     .update({"altitude": sq.columns.altitude,
-                              "lastseen": sq.c.lastseen,
+                              "lastseen": sq.columns.lastseen,
-                              "version": sq.c.version,
+                              "version": sq.columns.version,
-                              "platform": sq.c.platform})
+                              "platform": sq.columns.platform})
 
     logger.info("Count of receivers who where updated: {}".format(upd))
 
     # add new receivers
-    empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \
+    empty_sq = app.session.query(ReceiverBeacon.name,
-                          .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name,
+                                 ReceiverBeacon.latitude,
-                                       ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \
+                                 ReceiverBeacon.longitude,
+                                 ReceiverBeacon.altitude,
+                                 last_receiver_beacon_sq.columns.firstseen,
+                                 last_receiver_beacon_sq.columns.lastseen,
+                                 ReceiverBeacon.version, ReceiverBeacon.platform) \
+                          .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
+                                       ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
                           .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \
                           .filter(Receiver.name == null()) \
                           .order_by(ReceiverBeacon.name)