Added DeviceInfo, fixes #53

alembic/versions/4ebfb325db6_added_deviceinfo.py

@@ -0,0 +1,62 @@
+"""Added DeviceInfo
+
+Revision ID: 4ebfb325db6
+Revises: 163f6213d3f
+Create Date: 2016-06-04 11:11:00.546524
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4ebfb325db6'
+down_revision = '163f6213d3f'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.execute("CREATE TABLE device_info AS SELECT * FROM device;")
+    op.create_index('ix_device_info_address', 'device_info', ['address'])
+    op.drop_column('device_info', 'name')
+    op.drop_column('device_info', 'airport')
+    op.drop_column('device_info', 'frequency')
+
+    op.drop_column('device', 'address_origin')
+    op.drop_column('device', 'name')
+    op.drop_column('device', 'airport')
+    op.drop_column('device', 'aircraft')
+    op.drop_column('device', 'registration')
+    op.drop_column('device', 'competition')
+    op.drop_column('device', 'frequency')
+    op.drop_column('device', 'tracked')
+    op.drop_column('device', 'identified')
+
+    op.add_column('device', sa.Column('stealth', sa.Boolean))
+    op.add_column('device', sa.Column('software_version', sa.Float))
+    op.add_column('device', sa.Column('hardware_version', sa.SmallInteger))
+    op.add_column('device', sa.Column('real_address', sa.String(6)))
+
+
+def downgrade():
+    op.add_column('device', sa.Column('address_origin', sa.SmallInteger))
+    op.add_column('device', sa.Column('name', sa.Unicode))
+    op.add_column('device', sa.Column('airport', sa.String))
+    op.add_column('device', sa.Column('aircraft', sa.String))
+    op.add_column('device', sa.Column('registration', sa.String(7)))
+    op.add_column('device', sa.Column('competition', sa.String(3)))
+    op.add_column('device', sa.Column('frequency', sa.String))
+    op.add_column('device', sa.Column('tracked', sa.Boolean))
+    op.add_column('device', sa.Column('identified', sa.Boolean))
+
+    op.create_index('ix_device_info_registration', 'device', ['registration'])
+
+    op.drop_column('device', 'stealth')
+    op.drop_column('device', 'software_version')
+    op.drop_column('device', 'hardware_version')
+    op.drop_column('device', 'real_address')
+
+    # transfer from device_info to device costs too much...
+    op.execute("DROP TABLE device_info;")
+    pass

ogn/collect/database.py

@@ -1,8 +1,6 @@
-from sqlalchemy.sql import null
-
 from celery.utils.log import get_task_logger
 
-from ogn.model import Device, AddressOrigin
+from ogn.model import DeviceInfo, AddressOrigin
 from ogn.utils import get_ddb
 
 from ogn.collect.celery import app
@@ -10,45 +8,16 @@ from ogn.collect.celery import app
 
 logger = get_task_logger(__name__)
 
-temp_address_origin = 7
 
-
+def update_device_infos(session, address_origin, device_infos):
-def add_devices(session, origin):
+    session.query(DeviceInfo) \
-    before_sq = session.query(Device.address) \
+        .filter(DeviceInfo.address_origin == address_origin) \
-        .filter(Device.address_origin == origin) \
-        .subquery()
-    add_query = session.query(Device) \
-        .filter(Device.address_origin == temp_address_origin) \
-        .filter(~Device.address.in_(before_sq))
-
-    result = add_query.update({Device.address_origin: origin},
-                              synchronize_session='fetch')
-
-    return result
-
-
-def update_devices(session, origin, devices):
-    session.query(Device) \
-        .filter(Device.address_origin == temp_address_origin) \
-        .delete()
-
-    session.bulk_save_objects(devices)
-
-    # mark temporary added devices
-    session.query(Device) \
-        .filter(Device.address_origin == null()) \
-        .update({Device.address_origin: temp_address_origin})
-
-    logger.info('Added {} devices'.format(add_devices(session, origin)))
-
-    # delete temporary added devices
-    session.query(Device) \
-        .filter(Device.address_origin == temp_address_origin) \
         .delete()
 
+    session.bulk_save_objects(device_infos)
     session.commit()
 
-    return len(devices)
+    return len(device_infos)
 
 
 @app.task
@@ -56,7 +25,8 @@ def import_ddb():
     """Import registered devices from the DDB."""
 
     logger.info("Import registered devices fom the DDB...")
-    counter = update_devices(app.session, AddressOrigin.ogn_ddb, get_ddb())
+    counter = update_device_infos(app.session, AddressOrigin.ogn_ddb,
+                                  get_ddb())
     logger.info("Imported {} devices.".format(counter))
 
 
@@ -65,6 +35,6 @@ def import_file(path='tests/custom_ddb.txt'):
     """Import registered devices from a local file."""
 
     logger.info("Import registered devices from '{}'...".format(path))
-    counter = update_devices(app.session, AddressOrigin.user_defined,
+    counter = update_device_infos(app.session, AddressOrigin.user_defined,
-                             get_ddb(path))
+                                  get_ddb(path))
     logger.info("Imported {} devices.".format(counter))

ogn/commands/database.py

@@ -1,7 +1,7 @@
 from ogn.commands.dbutils import engine, session
 from ogn.model import Base, AddressOrigin
 from ogn.utils import get_ddb, get_airports
-from ogn.collect.database import update_devices
+from ogn.collect.database import update_device_infos
 
 from manager import Manager
 manager = Manager()
@@ -50,7 +50,9 @@ def import_ddb():
     """Import registered devices from the DDB."""
 
     print("Import registered devices fom the DDB...")
-    counter = update_devices(session, AddressOrigin.ogn_ddb, get_ddb())
+    address_origin = AddressOrigin.ogn_ddb
+    counter = update_device_infos(session, address_origin,
+                                  get_ddb(address_origin=address_origin))
     print("Imported %i devices." % counter)
 
 
@@ -60,8 +62,9 @@ def import_file(path='tests/custom_ddb.txt'):
     # (flushes previously manually imported entries)
 
     print("Import registered devices from '{}'...".format(path))
-    counter = update_devices(session, AddressOrigin.user_defined,
+    address_origin = AddressOrigin.user_defined
-                             get_ddb(path))
+    counter = update_device_infos(session, address_origin,
+                                  get_ddb(csvfile=path, address_origin=address_origin))
     print("Imported %i devices." % counter)
 
 

ogn/gateway/process.py

@@ -54,17 +54,25 @@ def process_beacon(raw_message):
             beacon = AircraftBeacon(**message)
 
             # connect beacon with device
-            device = session.query(Device.id) \
+            device = session.query(Device) \
                 .filter(Device.address == beacon.address) \
-                .order_by(Device.address_origin) \
                 .first()
             if device is None:
                 device = Device()
                 device.address = beacon.address
-                device.address_origin = AddressOrigin.seen
                 session.add(device)
             beacon.device_id = device.id
 
+            # update device
+            device.aircraft_type = beacon.aircraft_type
+            device.stealth = beacon.stealth
+            if beacon.hardware_version is not None:
+                device.hardware_version = beacon.hardware_version
+            if beacon.software_version is not None:
+                device.software_version = beacon.software_version
+            if beacon.real_address is not None:
+                device.real_address = beacon.real_address
+
             # connect beacon with receiver
             receiver = session.query(Receiver.id) \
                 .filter(Receiver.name == beacon.receiver_name) \

ogn/model/__init__.py

@@ -4,6 +4,7 @@ from .aircraft_type import AircraftType
 from .base import Base
 from .beacon import Beacon
 from .device import Device
+from .device_info import DeviceInfo
 from .aircraft_beacon import AircraftBeacon
 from .receiver_beacon import ReceiverBeacon
 from .receiver import Receiver

ogn/model/address_origin.py

@@ -1,22 +1,22 @@
 class AddressOrigin:
+    unknown = 0
     ogn_ddb = 1
     flarmnet = 2
     user_defined = 3
-    seen = 4
 
     def __init__(self, origin):
-        if origin in [1, 2, 3, 4]:
+        if origin in [0, 1, 2, 3]:
             self.origin = origin
         else:
             raise ValueError('no address origin with id {} known'.format(origin))
 
     def name(self):
-        if self.origin == self.ogn_ddb:
+        if self.origin == self.unknown:
+            return 'unknown'
+        elif self.origin == self.ogn_ddb:
             return 'OGN-DDB'
         elif self.origin == self.flarmnet:
             return 'FlarmNet'
         elif self.origin == self.user_defined:
             return 'user-defined'
-        elif self.origin == self.seen:
-            return 'seen'
         return ''

ogn/model/device.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, Unicode, Boolean, SmallInteger
+from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger
 from sqlalchemy.orm import relationship
 
 from .base import Base
@@ -8,32 +8,21 @@ class Device(Base):
     __tablename__ = 'device'
 
     id = Column(Integer, primary_key=True)
-    address_type = None
     address = Column(String(6), index=True)
-    name = Column(Unicode)
-    airport = Column(String)
-    aircraft = Column(String)
-    registration = Column(String(7), index=True)
-    competition = Column(String(3))
-    frequency = Column(String)
-    tracked = Column(Boolean)
-    identified = Column(Boolean)
     aircraft_type = Column(SmallInteger, index=True)
-
+    stealth = Column(Boolean)
-    address_origin = Column(SmallInteger)
+    software_version = Column(Float)
+    hardware_version = Column(SmallInteger)
+    real_address = Column(String(6))
 
     # Relations
     aircraft_beacons = relationship('AircraftBeacon')
 
     def __repr__(self):
-        return "<Device: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+        return "<Device: %s,%s,%s,%s,%s,%s>" % (
-            self.address_type,
             self.address,
-            self.name,
+            self.aircraft_type,
-            self.airport,
+            self.stealth,
-            self.aircraft,
+            self.software_version,
-            self.registration,
+            self.hardware_version,
-            self.competition,
+            self.real_address)
-            self.frequency,
-            self.tracked,
-            self.identified)

ogn/model/device_info.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String, Boolean, SmallInteger
+
+from .base import Base
+
+
+class DeviceInfo(Base):
+    __tablename__ = 'device_info'
+
+    id = Column(Integer, primary_key=True)
+    address_type = None
+    address = Column(String(6), index=True)
+    aircraft = Column(String)
+    registration = Column(String(7))
+    competition = Column(String(3))
+    tracked = Column(Boolean)
+    identified = Column(Boolean)
+    aircraft_type = Column(SmallInteger)
+
+    address_origin = Column(SmallInteger)
+
+    def __repr__(self):
+        return "<DeviceInfo: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
+            self.address_type,
+            self.address,
+            self.name,
+            self.airport,
+            self.aircraft,
+            self.registration,
+            self.competition,
+            self.frequency,
+            self.tracked,
+            self.identified)

ogn/utils.py

@@ -2,7 +2,7 @@ import requests
 import csv
 from io import StringIO
 
-from .model import Device, Airport, Location
+from .model import AddressOrigin, DeviceInfo, Airport, Location
 
 from geopy.geocoders import Nominatim
 from geopy.exc import GeopyError
@@ -21,7 +21,7 @@ nm2m = 1852
 mi2m = 1609.34
 
 
-def get_ddb(csvfile=None):
+def get_ddb(csvfile=None, address_origin=AddressOrigin.unknown):
     if csvfile is None:
         r = requests.get(DDB_URL)
         rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
@@ -31,21 +31,22 @@ def get_ddb(csvfile=None):
 
     data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
 
-    devices = list()
+    device_infos = list()
     for row in data:
-        device = Device()
+        device_info = DeviceInfo()
-        device.address_type = row[0]
+        device_info.address_type = row[0]
-        device.address = row[1]
+        device_info.address = row[1]
-        device.aircraft = row[2]
+        device_info.aircraft = row[2]
-        device.registration = row[3]
+        device_info.registration = row[3]
-        device.competition = row[4]
+        device_info.competition = row[4]
-        device.tracked = row[5] == 'Y'
+        device_info.tracked = row[5] == 'Y'
-        device.identified = row[6] == 'Y'
+        device_info.identified = row[6] == 'Y'
-        device.aircraft_type = int(row[7])
+        device_info.aircraft_type = int(row[7])
+        device_info.address_origin = address_origin
 
-        devices.append(device)
+        device_infos.append(device_info)
 
-    return devices
+    return device_infos
 
 
 def get_trackable(ddb):