diff --git a/README.md b/README.md index 554df50..9b0bbb2 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,8 @@ To load a custom configuration, create a file `myconfig.py` (see [config/default and set the environment variable `OGN_CONFIG_MODULE` accordingly. ``` -export OGN_CONFIG_MODULE="myconfig.py" +touch myconfig.py +export OGN_CONFIG_MODULE="myconfig" ./manage.py gateway.run ``` @@ -147,17 +148,20 @@ Only the command `logbook.compute` requires a running task server (celery) at th ### Available tasks -- ogn.collect.database - - `import_ddb` - Import registered devices from the ddb - - `import_file` - Import registered devices from a local file -- ogn.collect.receiver - - `update_receivers` - Populate/update receiver table -- ogn.collect.logbook - - `compute_takeoff_and_landing` - Generate TakeoffLanding table -- ogn.collect.heatmap - - `update_beacon_receiver_distance_all` - Calculate the distance between aircraft and - receiver for the last aircraft beacons +- `ogn.collect.database.import_ddb` - Import registered devices from the ddb +- `ogn.collect.database.import_file` - Import registered devices from a local file +- `ogn.collect.heatmap.update_beacon_receiver_distance_all` - Calculate the distance between aircraft and receiver for the last aircraft beacons +- `ogn.collect.receiver.update_receivers` - Populate/update receiver table (requires postgresql-backend) +- `ogn.collect.logbook.compute_takeoff_and_landing` - Generate TakeoffLanding table (requires postgresql-backend) + +If the task server is up and running, tasks could be started manually. + +``` +python3 +>>>from ogn.collect.database import import_ddb +>>>import_ddb.delay() +``` ## License Licensed under the [AGPLv3](LICENSE). diff --git a/config/default.py b/config/default.py index 6a0e3c8..5ef98b6 100644 --- a/config/default.py +++ b/config/default.py @@ -7,14 +7,23 @@ CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' from datetime import timedelta CELERYBEAT_SCHEDULE = { + 'update-ddb': { + 'task': 'ogn.collect.database.import_ddb', + 'schedule': timedelta(minutes=15), + }, 'update-receiver-distance': { 'task': 'ogn.collect.heatmap.update_beacon_receiver_distance_all', 'schedule': timedelta(minutes=5), }, - 'update-receiver-table': { - 'task': 'ogn.collect.receiver.update_receivers', - 'schedule': timedelta(minutes=5), - }, +# Only supported with postgresql backend +# 'update-logbook': { +# 'task': 'ogn.collect.logbook.compute_takeoff_and_landing', +# 'schedule': timedelta(minutes=15), +# }, +# 'update-receiver-table': { +# 'task': 'ogn.collect.receiver.update_receivers', +# 'schedule': timedelta(minutes=5), +# }, } CELERY_TIMEZONE = 'UTC' diff --git a/ogn/collect/receiver.py b/ogn/collect/receiver.py index 10b10cd..b2a5317 100644 --- a/ogn/collect/receiver.py +++ b/ogn/collect/receiver.py @@ -18,44 +18,58 @@ def update_receivers(): last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ .subquery() - last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, func.min(ReceiverBeacon.timestamp).label('firstseen'), func.max(ReceiverBeacon.timestamp).label('lastseen')) \ - .filter(ReceiverBeacon.timestamp >= last_entry_sq.c.last_entry) \ - .group_by(ReceiverBeacon.name) \ - .subquery() + last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, + func.min(ReceiverBeacon.timestamp).label('firstseen'), + func.max(ReceiverBeacon.timestamp).label('lastseen')) \ + .filter(ReceiverBeacon.timestamp >= last_entry_sq.columns.last_entry) \ + .group_by(ReceiverBeacon.name) \ + .subquery() # update existing receivers - sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ - .subquery() + sq = app.session.query(ReceiverBeacon.name, + ReceiverBeacon.latitude, + ReceiverBeacon.longitude, + ReceiverBeacon.altitude, + last_receiver_beacon_sq.columns.firstseen, + last_receiver_beacon_sq.columns.lastseen, + ReceiverBeacon.version, + ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, + ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \ + .subquery() - # -set country code to None if lat or lon changed + # set country code to None if lat or lon changed upd = app.session.query(Receiver) \ - .filter(and_(Receiver.name == sq.c.name, - or_(Receiver.latitude != sq.c.latitude, - Receiver.longitude != sq.c.longitude) - ) - ) \ - .update({"latitude": sq.c.latitude, - "longitude": sq.c.longitude, - "country_code": None}) + .filter(and_(Receiver.name == sq.columns.name, + or_(Receiver.latitude != sq.columns.latitude, + Receiver.longitude != sq.columns.longitude))) \ + .update({"latitude": sq.columns.latitude, + "longitude": sq.columns.longitude, + "country_code": ""}) logger.info("Count of receivers who changed lat or lon: {}".format(upd)) app.session.commit() - # -update lastseen of known receivers + # update lastseen of known receivers upd = app.session.query(Receiver) \ - .filter(Receiver.name == sq.c.name) \ - .update({"altitude": sq.c.altitude, - "lastseen": sq.c.lastseen, - "version": sq.c.version, - "platform": sq.c.platform}) + .filter(Receiver.name == sq.columns.name) \ + .update({"altitude": sq.columns.altitude, + "lastseen": sq.columns.lastseen, + "version": sq.columns.version, + "platform": sq.columns.platform}) logger.info("Count of receivers who where updated: {}".format(upd)) # add new receivers - empty_sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude, ReceiverBeacon.longitude, ReceiverBeacon.altitude, last_receiver_beacon_sq.c.firstseen, last_receiver_beacon_sq.c.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ - .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.c.name, - ReceiverBeacon.timestamp == last_receiver_beacon_sq.c.lastseen)) \ + empty_sq = app.session.query(ReceiverBeacon.name, + ReceiverBeacon.latitude, + ReceiverBeacon.longitude, + ReceiverBeacon.altitude, + last_receiver_beacon_sq.columns.firstseen, + last_receiver_beacon_sq.columns.lastseen, + ReceiverBeacon.version, ReceiverBeacon.platform) \ + .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, + ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \ .outerjoin(Receiver, Receiver.name == ReceiverBeacon.name) \ .filter(Receiver.name == null()) \ .order_by(ReceiverBeacon.name)