CLI: Remove celery dependency for simple tasks

Only `logbook.compute` still requires a running celery backend.

The celery tasks 'update_ddb_from_*' were renamed to
'database.import_ddb' and 'database.import_file'.

README.md

@@ -58,10 +58,10 @@ optional arguments:
 available commands:
 
   [db]
+    import_ddb             Import registered devices from the DDB.
+    import_file            Import registered devices from local file.
     init                   Initialize the database.
-    update_ddb_file        Update devices with data from local file.
+
-    update_ddb_ogn         Update devices with data from ogn.
-  
   [gateway]
     run                    Run the aprs client.
 
@@ -78,7 +78,7 @@ available commands:
     software_stats         Show some statistics of receiver software.
 ```
 
-The task server must be running for `db.updateddb`.
+Only the command `logbook.compute` requires a running task server (celery) at the moment.
 
 ## TODO
 - [x] Write celery backend and add task 'fetchddb'
@@ -95,9 +95,13 @@ The task server must be running for `db.updateddb`.
 - [ ] Introduce scheduled tasks with 'celery beat' (eg. updateddb)
 
 ### Scheduled tasks
-- ogn.collect.fetchddb (generate Flarm table)
+- ogn.collect.database
-- ogn.collect.receiver (generate Receiver table)
+  - import_ddb - Import registered devices from the ddb
-- ogn.collect.logbook  (generate TakeoffLanding table)
+  - import_file - Import registered devices from a local file
+- ogn.collect.receiver
+  - populate - generate Receiver table (not implemented)
+- ogn.collect.logbook
+  - compute - generate TakeoffLanding table
 
 ## How to use virtualenv
 ```

ogn/collect/celery.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
@@ -9,7 +7,7 @@ from celery.signals import worker_init, worker_shutdown
 app = Celery('ogn.collect',
              broker='redis://localhost:6379/0',
              backend='redis://localhost:6379/0',
-             include=["ogn.collect.fetchddb", "ogn.collect.logbook"])
+             include=["ogn.collect.database", "ogn.collect.logbook"])
 
 DB_URI = 'sqlite:///beacons.db'
 

ogn/collect/database.py

@@ -0,0 +1,35 @@
+from ogn.model import Device
+
+from celery.utils.log import get_task_logger
+from ogn.collect.celery import app
+
+logger = get_task_logger(__name__)
+
+
+def update_devices(session, origin, devices):
+    session.query(Device) \
+        .filter(Device.address_origin == origin) \
+        .delete()
+
+    session.bulk_save_objects(devices)
+    session.commit()
+
+    return len(devices)
+
+
+@app.task
+def import_ddb():
+    """Import registered devices from the DDB."""
+
+    logger.info("Import registered devices fom the DDB...")
+    counter = update_devices(app.session, AddressOrigin.ogn_ddb, get_ddb())
+    logger.info("Imported %i devices." % counter)
+
+
+@app.task
+def import_file(path='tests/custom_ddb.txt'):
+    """Import registered devices from a local file."""
+
+    logger.info("Import registered devices from '{}'...".format(path))
+    counter = update_devices(app.session, AddressOrigin.user_defined, get_ddb(path))
+    logger.info("Imported %i devices." % counter)

ogn/collect/fetchddb.py

@@ -1,43 +0,0 @@
-from __future__ import absolute_import
-
-from celery.utils.log import get_task_logger
-from ogn.collect.celery import app
-
-from ogn.model import AddressOrigin, Device
-from ogn.utils import get_ddb
-
-logger = get_task_logger(__name__)
-
-
-@app.task
-def update_ddb_from_ogn():
-    logger.info("Update ddb data from ogn.")
-
-    app.session.query(Device) \
-        .filter(Device.address_origin == AddressOrigin.ogn_ddb) \
-        .delete()
-
-    devices = get_ddb()
-    logger.debug("New Devices: %s" % str(devices))
-
-    app.session.bulk_save_objects(devices)
-    app.session.commit()
-
-    return len(devices)
-
-
-@app.task
-def update_ddb_from_file():
-    logger.info("Import ddb data from file.")
-
-    app.session.query(Device) \
-        .filter(Device.address_origin == AddressOrigin.userdefined) \
-        .delete()
-
-    devices = get_ddb('ogn/custom_ddb.txt')
-    logger.debug("New Devices: %s" % str(devices))
-
-    app.session.bulk_save_objects(devices)
-    app.session.commit()
-
-    return len(devices)

ogn/collect/logbook.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from datetime import datetime, timedelta
 
 from celery.utils.log import get_task_logger

ogn/commands/database.py

@@ -1,8 +1,7 @@
-from sqlalchemy import func, and_, true, false
-
-from ogn.model import Base, AddressOrigin, Device
-from ogn.collect.fetchddb import update_ddb_from_ogn, update_ddb_from_file
 from ogn.commands.dbutils import engine, session
+from ogn.model import Base, AddressOrigin
+from ogn.utils import get_ddb
+from ogn.collect.database import update_devices
 
 from manager import Manager
 manager = Manager()
@@ -17,22 +16,19 @@ def init():
 
 
 @manager.command
-def update_ddb_ogn():
+def import_ddb():
-    """Update devices with data from ogn."""
+    """Import registered devices from the DDB."""
-    print("Updating ddb data...")
+
-    result = update_ddb_from_ogn.delay()
+    print("Import registered devices fom the DDB...")
-    counter = result.get()
+    counter = update_devices(session, AddressOrigin.ogn_ddb, get_ddb())
     print("Imported %i devices." % counter)
 
 
 @manager.command
-def update_ddb_file():
+def import_file(path='tests/custom_ddb.txt'):
-    """Update devices with data from local file."""
+    """Import registered devices from a local file."""
-    print("Updating ddb data...")
+    # (flushes previously manually imported entries)
-    result = update_ddb_from_file.delay()
+
-    counter = result.get()
+    print("Import registered devices from '{}'...".format(path))
+    counter = update_devices(session, AddressOrigin.user_defined, get_ddb(path))
     print("Imported %i devices." % counter)
-
-
-@manager.command
-