Changed address origin handling

ogn/collect/database.py

@@ -1,3 +1,5 @@
+from sqlalchemy.sql import null
+
 from celery.utils.log import get_task_logger
 
 from ogn.model import Device, AddressOrigin
@@ -8,13 +10,42 @@ from ogn.collect.celery import app
 
 logger = get_task_logger(__name__)
 
+temp_address_origin = 7
+
+
+def add_devices(session, origin):
+    before_sq = session.query(Device.address) \
+        .filter(Device.address_origin == origin) \
+        .subquery()
+    add_query = session.query(Device) \
+        .filter(Device.address_origin == temp_address_origin) \
+        .filter(~Device.address.in_(before_sq))
+
+    result = add_query.update({Device.address_origin: origin},
+                              synchronize_session='fetch')
+
+    return result
+
 
 def update_devices(session, origin, devices):
     session.query(Device) \
-        .filter(Device.address_origin == origin) \
+        .filter(Device.address_origin == temp_address_origin) \
         .delete()
 
     session.bulk_save_objects(devices)
+
+    # mark temporary added devices
+    session.query(Device) \
+        .filter(Device.address_origin == null()) \
+        .update({Device.address_origin: temp_address_origin})
+
+    logger.info('Added {} devices'.format(add_devices(session, origin)))
+
+    # delete temporary added devices
+    session.query(Device) \
+        .filter(Device.address_origin == temp_address_origin) \
+        .delete()
+
     session.commit()
 
     return len(devices)

ogn/model/address_origin.py

@@ -2,9 +2,10 @@ class AddressOrigin:
     ogn_ddb = 1
     flarmnet = 2
     user_defined = 3
+    seen = 4
 
     def __init__(self, origin):
-        if origin in [1, 2, 3]:
+        if origin in [1, 2, 3, 4]:
             self.origin = origin
         else:
             raise ValueError('no address origin with id {} known'.format(origin))
@@ -16,4 +17,6 @@ class AddressOrigin:
             return 'FlarmNet'
         elif self.origin == self.user_defined:
             return 'user-defined'
+        elif self.origin == self.seen:
+            return 'seen'
         return ''

ogn/utils.py

@@ -2,7 +2,7 @@ import requests
 import csv
 from io import StringIO
 
-from .model import Device, AddressOrigin, Airport, Location
+from .model import Device, Airport, Location
 
 from geopy.geocoders import Nominatim
 from geopy.exc import GeopyError
@@ -25,28 +25,24 @@ def get_ddb(csvfile=None):
     if csvfile is None:
         r = requests.get(DDB_URL)
         rows = '\n'.join(i for i in r.text.splitlines() if i[0] != '#')
-        address_origin = AddressOrigin.ogn_ddb
     else:
         r = open(csvfile, 'r')
         rows = ''.join(i for i in r.readlines() if i[0] != '#')
-        address_origin = AddressOrigin.user_defined
 
     data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
 
     devices = list()
     for row in data:
-        flarm = Device()
-        flarm.address_type = row[0]
-        flarm.address = row[1]
-        flarm.aircraft = row[2]
-        flarm.registration = row[3]
-        flarm.competition = row[4]
-        flarm.tracked = row[5] == 'Y'
-        flarm.identified = row[6] == 'Y'
+        device = Device()
+        device.address_type = row[0]
+        device.address = row[1]
+        device.aircraft = row[2]
+        device.registration = row[3]
+        device.competition = row[4]
+        device.tracked = row[5] == 'Y'
+        device.identified = row[6] == 'Y'
 
-        flarm.address_origin = address_origin
-
-        devices.append(flarm)
+        devices.append(device)
 
     return devices
 
@@ -77,6 +73,9 @@ def get_airports(cupfile):
         for line in f:
             try:
                 for waypoint in Reader([line]):
+                    if waypoint['style'] > 5:   # reject unlandable places
+                        continue
+
                     airport = Airport()
                     airport.name = waypoint['name']
                     airport.code = waypoint['code']