From lat/lon to POINT

Flake8 suggestions

Add shapely

Fixed update_receivers

migrate table takeoff_landing

alembic/versions/277aca1b810_migrate_to_postgis.py

@@ -0,0 +1,85 @@
+"""Migrate to PostGIS
+
+Revision ID: 277aca1b810
+Revises: 3a0765c9a2
+Create Date: 2016-04-23 08:01:49.059187
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '277aca1b810'
+down_revision = '3a0765c9a2'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+import geoalchemy2 as ga
+
+UPGRADE_QUERY = """
+UPDATE {table_name}
+SET
+    location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326);
+"""
+
+DOWNGRADE_QUERY = """
+UPDATE {table_name}
+SET
+    latitude = ST_Y(ST_TRANSFORM(location, 4326)),
+    longitude = ST_X(ST_TRANSFORM(location, 4326));
+"""
+
+def upgrade():
+    #CREATE EXTENSION IF NOT EXISTS postgis
+    op.add_column('airport', sa.Column('location', ga.Geometry('POINT', srid=4326)))
+    op.execute(UPGRADE_QUERY.format(table_name='airport'))
+    op.drop_column('airport', 'latitude')
+    op.drop_column('airport', 'longitude')
+
+    op.add_column('aircraft_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
+    op.execute(UPGRADE_QUERY.format(table_name='aircraft_beacon'))
+    op.drop_column('aircraft_beacon', 'latitude')
+    op.drop_column('aircraft_beacon', 'longitude')
+
+    op.add_column('receiver_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
+    op.execute(UPGRADE_QUERY.format(table_name='receiver_beacon'))
+    op.drop_column('receiver_beacon', 'latitude')
+    op.drop_column('receiver_beacon', 'longitude')
+
+    op.add_column('receiver', sa.Column('location', ga.Geometry('POINT', srid=4326)))
+    op.execute(UPGRADE_QUERY.format(table_name='receiver'))
+    op.drop_column('receiver', 'latitude')
+    op.drop_column('receiver', 'longitude')
+
+    op.add_column('takeoff_landing', sa.Column('location', ga.Geometry('POINT', srid=4326)))
+    op.execute(UPGRADE_QUERY.format(table_name='takeoff_landing'))
+    op.drop_column('takeoff_landing', 'latitude')
+    op.drop_column('takeoff_landing', 'longitude')
+
+
+def downgrade():
+    #DROP EXTENSION postgis
+    op.add_column('airport', sa.Column('latitude', sa.FLOAT))
+    op.add_column('airport', sa.Column('longitude', sa.FLOAT))
+    op.execute(DOWNGRADE_QUERY.format(table_name='airport'))
+    op.drop_column('airport', 'location')
+
+    op.add_column('aircraft_beacon', sa.Column('latitude', sa.FLOAT))
+    op.add_column('aircraft_beacon', sa.Column('longitude', sa.FLOAT))
+    op.execute(DOWNGRADE_QUERY.format(table_name='aircraft_beacon'))
+    op.drop_column('aircraft_beacon', 'location')
+
+    op.add_column('receiver_beacon', sa.Column('latitude', sa.FLOAT))
+    op.add_column('receiver_beacon', sa.Column('longitude', sa.FLOAT))
+    op.execute(DOWNGRADE_QUERY.format(table_name='receiver_beacon'))
+    op.drop_column('receiver_beacon', 'location')
+
+    op.add_column('receiver', sa.Column('latitude', sa.FLOAT))
+    op.add_column('receiver', sa.Column('longitude', sa.FLOAT))
+    op.execute(DOWNGRADE_QUERY.format(table_name='receiver'))
+    op.drop_column('receiver', 'location')
+
+    op.add_column('takeoff_landing', sa.Column('latitude', sa.FLOAT))
+    op.add_column('takeoff_landing', sa.Column('longitude', sa.FLOAT))
+    op.execute(DOWNGRADE_QUERY.format(table_name='takeoff_landing'))
+    op.drop_column('takeoff_landing', 'location')

ogn/collect/logbook.py

@@ -41,13 +41,9 @@ def compute_takeoff_and_landing():
         AircraftBeacon.timestamp,
         func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
         func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
-        AircraftBeacon.latitude,
+        AircraftBeacon.location_wkt,
-        func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
+        func.lag(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_prev'),
-        func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
+        func.lead(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_next'),
-        AircraftBeacon.longitude,
-        func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
-        func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
-        AircraftBeacon.ground_speed,
         AircraftBeacon.track,
         func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
         func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
@@ -65,8 +61,7 @@ def compute_takeoff_and_landing():
     takeoff_landing_query = app.session.query(
         sq.c.address,
         sq.c.timestamp,
-        sq.c.latitude,
+        sq.c.location,
-        sq.c.longitude,
         sq.c.track,
         sq.c.ground_speed,
         sq.c.altitude,
@@ -82,7 +77,7 @@ def compute_takeoff_and_landing():
         .order_by(func.date(sq.c.timestamp), sq.c.timestamp)
 
     # ... and save them
-    ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
+    ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.location_wkt, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
     result = app.session.execute(ins)
     counter = result.rowcount
     app.session.commit()

ogn/collect/receiver.py

@@ -1,6 +1,6 @@
 from sqlalchemy.sql import func, null
 from sqlalchemy.sql.functions import coalesce
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, not_
 
 from celery.utils.log import get_task_logger
 
@@ -26,8 +26,7 @@ def update_receivers():
                                          .subquery()
 
     receivers_to_update = app.session.query(ReceiverBeacon.name,
-                                            ReceiverBeacon.latitude,
+                                            ReceiverBeacon.location_wkt,
-                                            ReceiverBeacon.longitude,
                                             ReceiverBeacon.altitude,
                                             last_receiver_beacon_sq.columns.lastseen,
                                             ReceiverBeacon.version,
@@ -39,11 +38,10 @@ def update_receivers():
     # set country code to None if lat or lon changed
     count = app.session.query(Receiver) \
                        .filter(and_(Receiver.name == receivers_to_update.columns.name,
-                                    or_(Receiver.latitude != receivers_to_update.columns.latitude,
+                                    not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)))) \
-                                        Receiver.longitude != receivers_to_update.columns.longitude))) \
+                       .update({"location_wkt": receivers_to_update.columns.location,
-                       .update({"latitude": receivers_to_update.columns.latitude,
+                                "country_code": null()},
-                                "longitude": receivers_to_update.columns.longitude,
+                               synchronize_session=False)
-                                "country_code": null()})
 
     logger.info("Count of receivers who changed lat or lon: {}".format(count))
 
@@ -59,8 +57,7 @@ def update_receivers():
 
     # add new receivers
     empty_sq = app.session.query(ReceiverBeacon.name,
-                                 ReceiverBeacon.latitude,
+                                 ReceiverBeacon.location_wkt,
-                                 ReceiverBeacon.longitude,
                                  ReceiverBeacon.altitude,
                                  last_receiver_beacon_sq.columns.lastseen,
                                  ReceiverBeacon.version, ReceiverBeacon.platform) \
@@ -73,8 +70,7 @@ def update_receivers():
     for receiver_beacon in empty_sq.all():
         receiver = Receiver()
         receiver.name = receiver_beacon.name
-        receiver.latitude = receiver_beacon.latitude
+        receiver.location_wkt = receiver_beacon.location_wkt
-        receiver.longitude = receiver_beacon.longitude
         receiver.altitude = receiver_beacon.altitude
         receiver.firstseen = None
         receiver.lastseen = receiver_beacon.lastseen
@@ -103,7 +99,8 @@ def update_receivers():
                                        .order_by(Receiver.name)
 
     for receiver in unknown_country_query.all():
-        receiver.country_code = get_country_code(receiver.latitude, receiver.longitude)
+        location = receiver.location
+        receiver.country_code = get_country_code(location.latitude, location.longitude)
         if receiver.country_code is not None:
             logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
 

ogn/commands/logbook.py

@@ -3,7 +3,7 @@
 from datetime import timedelta
 
 from sqlalchemy.sql import func, null
-from sqlalchemy import and_, or_, between
+from sqlalchemy import and_, or_
 from sqlalchemy.sql.expression import true, false, label
 
 from ogn.model import Device, TakeoffLanding, Airport
@@ -28,21 +28,15 @@ def compute():
 def show(airport_name):
     """Show a logbook for <airport_name>."""
     airport = session.query(Airport) \
-        .filter(or_(Airport.name==airport_name)) \
+        .filter(Airport.name == airport_name) \
         .first()
 
     if (airport is None):
         print('Airport "{}" not found.'.format(airport_name))
         return
 
-    latitude = float(airport.latitude)
+    delta_altitude = 200
-    longitude = float(airport.longitude)
+    delta_radius = 10
-    altitude = float(airport.altitude)
-    latmin = latitude - 0.05
-    latmax = latitude + 0.05
-    lonmin = longitude - 0.05
-    lonmax = longitude + 0.05
-    max_altitude = altitude + 200
 
     # make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
     sq = session.query(
@@ -91,9 +85,8 @@ def show(airport_name):
                                     TakeoffLanding.address,
                                     TakeoffLanding.timestamp))
                 .label('is_takeoff_next')) \
-        .filter(and_(between(TakeoffLanding.latitude, latmin, latmax),
+        .filter(func.ST_DFullyWithin(TakeoffLanding.location_wkt, Airport.location_wkt, delta_radius)) \
-                     between(TakeoffLanding.longitude, lonmin, lonmax))) \
+        .filter(TakeoffLanding.altitude < Airport.altitude + delta_altitude) \
-        .filter(TakeoffLanding.altitude < max_altitude) \
         .subquery()
 
     # find complete flights (with takeoff and landing) with duration < 1 day

ogn/gateway/process.py

@@ -1,11 +1,19 @@
 import logging
 from ogn.commands.dbutils import session
-from ogn.model import AircraftBeacon, ReceiverBeacon
+from ogn.model import AircraftBeacon, ReceiverBeacon, Location
 from ogn.parser import parse_aprs, parse_ogn_receiver_beacon, parse_ogn_aircraft_beacon, ParseError
 
 logger = logging.getLogger(__name__)
 
 
+def replace_lonlat_with_wkt(message):
+    location = Location(message['longitude'], message['latitude'])
+    message['location_wkt'] = location.to_wkt()
+    del message['latitude']
+    del message['longitude']
+    return message
+
+
 def process_beacon(raw_message):
     if raw_message[0] == '#':
         return
@@ -25,9 +33,11 @@ def process_beacon(raw_message):
         # /o: ?
         if message['symboltable'] == "I" and message['symbolcode'] == '&':
             message.update(parse_ogn_receiver_beacon(message['comment']))
+            message = replace_lonlat_with_wkt(message)
             beacon = ReceiverBeacon(**message)
         else:
             message.update(parse_ogn_aircraft_beacon(message['comment']))
+            message = replace_lonlat_with_wkt(message)
             beacon = AircraftBeacon(**message)
         session.add(beacon)
         session.commit()
@@ -35,3 +45,5 @@ def process_beacon(raw_message):
     except ParseError as e:
         logger.error('Received message: {}'.format(raw_message))
         logger.error('Drop packet, {}'.format(e.message))
+    except TypeError as e:
+        logger.error('TypeError: {}'.format(raw_message))

ogn/model/__init__.py

@@ -9,3 +9,5 @@ from .receiver_beacon import ReceiverBeacon
 from .receiver import Receiver
 from .takeoff_landing import TakeoffLanding
 from .airport import Airport
+
+from .geo import Location

ogn/model/airport.py

@@ -1,4 +1,5 @@
 from sqlalchemy import Column, String, Integer, Float, SmallInteger
+from geoalchemy2.types import Geometry
 
 from .base import Base
 
@@ -8,14 +9,14 @@ class Airport(Base):
 
     id = Column(Integer, primary_key=True)
 
+    location_wkt = Column('location', Geometry('POINT', srid=4326))
+    altitude = Column(Integer)
+
     name = Column(String, index=True)
     code = Column(String(5))
     country_code = Column(String(2))
     style = Column(SmallInteger)
     description = Column(String)
-    latitude = Column(Float)
-    longitude = Column(Float)
-    altitude = Column(Integer)
     runway_direction = Column(Integer)
     runway_length = Column(Integer)
     frequency = Column(Float)

ogn/model/beacon.py

@@ -1,21 +1,32 @@
 from sqlalchemy import Column, String, Integer, Float, DateTime
 from sqlalchemy.ext.declarative import AbstractConcreteBase
+from geoalchemy2.types import Geometry
+from geoalchemy2.shape import to_shape
 
 from .base import Base
+from .geo import Location
 
 
 class Beacon(AbstractConcreteBase, Base):
     id = Column(Integer, primary_key=True)
 
     # APRS data
+    location_wkt = Column('location', Geometry('POINT', srid=4326))
+    altitude = Column(Integer)
+
     name = Column(String)
     receiver_name = Column(String(9))
     timestamp = Column(DateTime, index=True)
-    latitude = Column(Float)
     symboltable = None
-    longitude = Column(Float)
     symbolcode = None
     track = Column(Integer)
     ground_speed = Column(Float)
-    altitude = Column(Integer)
     comment = None
+
+    @property
+    def location(self):
+        if self.location_wkt is None:
+            return None
+
+        coords = to_shape(self.location_wkt)
+        return Location(lat=coords.y, lon=coords.x)

ogn/model/geo.py

@@ -0,0 +1,15 @@
+class Location:
+    """Represents a location in WGS84"""
+
+    def __init__(self, lon, lat):
+        self.longitude = lon
+        self.latitude = lat
+
+    def to_wkt(self):
+        return 'SRID=4326;POINT({0} {1})'.format(self.longitude, self.latitude)
+
+    def __str__(self):
+        return '{0: 7.4f}, {1:8.4f}'.format(self.latitude, self.longitude)
+
+    def as_dict(self):
+        return {'latitude': round(self.latitude, 8), 'longitude': round(self.longitude, 8)}

ogn/model/receiver.py

@@ -1,18 +1,30 @@
-from sqlalchemy import Column, String, Integer, Float, DateTime
+from sqlalchemy import Column, String, Integer, DateTime
+from geoalchemy2.types import Geometry
+from geoalchemy2.shape import to_shape
 
 from .base import Base
+from .geo import Location
 
 
 class Receiver(Base):
     __tablename__ = "receiver"
 
     id = Column(Integer, primary_key=True)
-    name = Column(String(9))
+
-    latitude = Column(Float)
+    location_wkt = Column('location', Geometry('POINT', srid=4326))
-    longitude = Column(Float)
     altitude = Column(Integer)
+
+    name = Column(String(9))
     firstseen = Column(DateTime, index=True)
     lastseen = Column(DateTime, index=True)
     country_code = Column(String(2))
     version = Column(String)
     platform = Column(String)
+
+    @property
+    def location(self):
+        if self.location_wkt is None:
+            return None
+
+        coords = to_shape(self.location_wkt)
+        return Location(lat=coords.y, lon=coords.x)

ogn/utils.py

@@ -2,7 +2,7 @@ import requests
 import csv
 from io import StringIO
 
-from .model import Device, AddressOrigin, Airport
+from .model import Device, AddressOrigin, Airport, Location
 
 from geopy.geocoders import Nominatim
 from geopy.exc import GeopyError
@@ -83,22 +83,22 @@ def get_airports(cupfile):
                     airport.country_code = waypoint['country']
                     airport.style = waypoint['style']
                     airport.description = waypoint['description']
-                    airport.latitude = waypoint['latitude']
+                    location = Location(waypoint['longitude'], waypoint['latitude'])
-                    airport.longitude = waypoint['longitude']
+                    airport.location_wkt = location.to_wkt()
                     airport.altitude = waypoint['elevation']['value']
                     if (waypoint['elevation']['unit'] == 'ft'):
-                        airport.altitude = airport.altitude*feet2m
+                        airport.altitude = airport.altitude * feet2m
                     airport.runway_direction = waypoint['runway_direction']
                     airport.runway_length = waypoint['runway_length']['value']
                     if (waypoint['runway_length']['unit'] == 'nm'):
-                        airport.altitude = airport.altitude*nm2m
+                        airport.altitude = airport.altitude * nm2m
                     elif (waypoint['runway_length']['unit'] == 'ml'):
-                        airport.altitude = airport.altitude*mi2m
+                        airport.altitude = airport.altitude * mi2m
                     airport.frequency = waypoint['frequency']
 
                     airports.append(airport)
-            except Exception:
+            except AttributeError as e:
-                print('Failed to parse line: {}'.format(line))
+                print('Failed to parse line: {} {}'.format(line, e))
 
     return airports
 
@@ -115,4 +115,3 @@ def haversine_distance(location0, location1):
     phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1)))
 
     return distance, phi
-

setup.py

@@ -38,6 +38,8 @@ setup(
         'celery[redis]>=3.1,<3.2',
         'alembic==0.8.3',
         'aerofiles==0.3',
+        'geoalchemy2==0.3.0',
+        'shapely==1.5.15',
         'ogn-client==0.3.0'
     ],
     extras_require={