diff --git a/alembic/versions/277aca1b810_migrate_to_postgis.py b/alembic/versions/277aca1b810_migrate_to_postgis.py new file mode 100644 index 0000000..edfcf7c --- /dev/null +++ b/alembic/versions/277aca1b810_migrate_to_postgis.py @@ -0,0 +1,85 @@ +"""Migrate to PostGIS + +Revision ID: 277aca1b810 +Revises: 3a0765c9a2 +Create Date: 2016-04-23 08:01:49.059187 + +""" + +# revision identifiers, used by Alembic. +revision = '277aca1b810' +down_revision = '3a0765c9a2' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +import geoalchemy2 as ga + +UPGRADE_QUERY = """ +UPDATE {table_name} +SET + location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326); +""" + +DOWNGRADE_QUERY = """ +UPDATE {table_name} +SET + latitude = ST_Y(ST_TRANSFORM(location, 4326)), + longitude = ST_X(ST_TRANSFORM(location, 4326)); +""" + +def upgrade(): + #CREATE EXTENSION IF NOT EXISTS postgis + op.add_column('airport', sa.Column('location', ga.Geometry('POINT', srid=4326))) + op.execute(UPGRADE_QUERY.format(table_name='airport')) + op.drop_column('airport', 'latitude') + op.drop_column('airport', 'longitude') + + op.add_column('aircraft_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326))) + op.execute(UPGRADE_QUERY.format(table_name='aircraft_beacon')) + op.drop_column('aircraft_beacon', 'latitude') + op.drop_column('aircraft_beacon', 'longitude') + + op.add_column('receiver_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326))) + op.execute(UPGRADE_QUERY.format(table_name='receiver_beacon')) + op.drop_column('receiver_beacon', 'latitude') + op.drop_column('receiver_beacon', 'longitude') + + op.add_column('receiver', sa.Column('location', ga.Geometry('POINT', srid=4326))) + op.execute(UPGRADE_QUERY.format(table_name='receiver')) + op.drop_column('receiver', 'latitude') + op.drop_column('receiver', 'longitude') + + op.add_column('takeoff_landing', sa.Column('location', ga.Geometry('POINT', srid=4326))) + op.execute(UPGRADE_QUERY.format(table_name='takeoff_landing')) + op.drop_column('takeoff_landing', 'latitude') + op.drop_column('takeoff_landing', 'longitude') + + +def downgrade(): + #DROP EXTENSION postgis + op.add_column('airport', sa.Column('latitude', sa.FLOAT)) + op.add_column('airport', sa.Column('longitude', sa.FLOAT)) + op.execute(DOWNGRADE_QUERY.format(table_name='airport')) + op.drop_column('airport', 'location') + + op.add_column('aircraft_beacon', sa.Column('latitude', sa.FLOAT)) + op.add_column('aircraft_beacon', sa.Column('longitude', sa.FLOAT)) + op.execute(DOWNGRADE_QUERY.format(table_name='aircraft_beacon')) + op.drop_column('aircraft_beacon', 'location') + + op.add_column('receiver_beacon', sa.Column('latitude', sa.FLOAT)) + op.add_column('receiver_beacon', sa.Column('longitude', sa.FLOAT)) + op.execute(DOWNGRADE_QUERY.format(table_name='receiver_beacon')) + op.drop_column('receiver_beacon', 'location') + + op.add_column('receiver', sa.Column('latitude', sa.FLOAT)) + op.add_column('receiver', sa.Column('longitude', sa.FLOAT)) + op.execute(DOWNGRADE_QUERY.format(table_name='receiver')) + op.drop_column('receiver', 'location') + + op.add_column('takeoff_landing', sa.Column('latitude', sa.FLOAT)) + op.add_column('takeoff_landing', sa.Column('longitude', sa.FLOAT)) + op.execute(DOWNGRADE_QUERY.format(table_name='takeoff_landing')) + op.drop_column('takeoff_landing', 'location') diff --git a/ogn/collect/logbook.py b/ogn/collect/logbook.py index a748821..99b5889 100644 --- a/ogn/collect/logbook.py +++ b/ogn/collect/logbook.py @@ -41,13 +41,9 @@ def compute_takeoff_and_landing(): AircraftBeacon.timestamp, func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'), func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'), - AircraftBeacon.latitude, - func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'), - func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'), - AircraftBeacon.longitude, - func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'), - func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'), - AircraftBeacon.ground_speed, + AircraftBeacon.location_wkt, + func.lag(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_prev'), + func.lead(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_next'), AircraftBeacon.track, func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'), func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'), @@ -65,8 +61,7 @@ def compute_takeoff_and_landing(): takeoff_landing_query = app.session.query( sq.c.address, sq.c.timestamp, - sq.c.latitude, - sq.c.longitude, + sq.c.location, sq.c.track, sq.c.ground_speed, sq.c.altitude, @@ -82,7 +77,7 @@ def compute_takeoff_and_landing(): .order_by(func.date(sq.c.timestamp), sq.c.timestamp) # ... and save them - ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query) + ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.location_wkt, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query) result = app.session.execute(ins) counter = result.rowcount app.session.commit() diff --git a/ogn/collect/receiver.py b/ogn/collect/receiver.py index e7a3565..e986597 100644 --- a/ogn/collect/receiver.py +++ b/ogn/collect/receiver.py @@ -1,6 +1,6 @@ from sqlalchemy.sql import func, null from sqlalchemy.sql.functions import coalesce -from sqlalchemy import and_, or_ +from sqlalchemy import and_, not_ from celery.utils.log import get_task_logger @@ -26,8 +26,7 @@ def update_receivers(): .subquery() receivers_to_update = app.session.query(ReceiverBeacon.name, - ReceiverBeacon.latitude, - ReceiverBeacon.longitude, + ReceiverBeacon.location_wkt, ReceiverBeacon.altitude, last_receiver_beacon_sq.columns.lastseen, ReceiverBeacon.version, @@ -39,11 +38,10 @@ def update_receivers(): # set country code to None if lat or lon changed count = app.session.query(Receiver) \ .filter(and_(Receiver.name == receivers_to_update.columns.name, - or_(Receiver.latitude != receivers_to_update.columns.latitude, - Receiver.longitude != receivers_to_update.columns.longitude))) \ - .update({"latitude": receivers_to_update.columns.latitude, - "longitude": receivers_to_update.columns.longitude, - "country_code": null()}) + not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)))) \ + .update({"location_wkt": receivers_to_update.columns.location, + "country_code": null()}, + synchronize_session=False) logger.info("Count of receivers who changed lat or lon: {}".format(count)) @@ -59,8 +57,7 @@ def update_receivers(): # add new receivers empty_sq = app.session.query(ReceiverBeacon.name, - ReceiverBeacon.latitude, - ReceiverBeacon.longitude, + ReceiverBeacon.location_wkt, ReceiverBeacon.altitude, last_receiver_beacon_sq.columns.lastseen, ReceiverBeacon.version, ReceiverBeacon.platform) \ @@ -73,8 +70,7 @@ def update_receivers(): for receiver_beacon in empty_sq.all(): receiver = Receiver() receiver.name = receiver_beacon.name - receiver.latitude = receiver_beacon.latitude - receiver.longitude = receiver_beacon.longitude + receiver.location_wkt = receiver_beacon.location_wkt receiver.altitude = receiver_beacon.altitude receiver.firstseen = None receiver.lastseen = receiver_beacon.lastseen @@ -103,7 +99,8 @@ def update_receivers(): .order_by(Receiver.name) for receiver in unknown_country_query.all(): - receiver.country_code = get_country_code(receiver.latitude, receiver.longitude) + location = receiver.location + receiver.country_code = get_country_code(location.latitude, location.longitude) if receiver.country_code is not None: logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) diff --git a/ogn/commands/logbook.py b/ogn/commands/logbook.py index 10a5237..18847d5 100644 --- a/ogn/commands/logbook.py +++ b/ogn/commands/logbook.py @@ -3,7 +3,7 @@ from datetime import timedelta from sqlalchemy.sql import func, null -from sqlalchemy import and_, or_, between +from sqlalchemy import and_, or_ from sqlalchemy.sql.expression import true, false, label from ogn.model import Device, TakeoffLanding, Airport @@ -28,21 +28,15 @@ def compute(): def show(airport_name): """Show a logbook for .""" airport = session.query(Airport) \ - .filter(or_(Airport.name==airport_name)) \ + .filter(Airport.name == airport_name) \ .first() if (airport is None): print('Airport "{}" not found.'.format(airport_name)) return - latitude = float(airport.latitude) - longitude = float(airport.longitude) - altitude = float(airport.altitude) - latmin = latitude - 0.05 - latmax = latitude + 0.05 - lonmin = longitude - 0.05 - lonmax = longitude + 0.05 - max_altitude = altitude + 200 + delta_altitude = 200 + delta_radius = 10 # make a query with current, previous and next "takeoff_landing" event, so we can find complete flights sq = session.query( @@ -91,9 +85,8 @@ def show(airport_name): TakeoffLanding.address, TakeoffLanding.timestamp)) .label('is_takeoff_next')) \ - .filter(and_(between(TakeoffLanding.latitude, latmin, latmax), - between(TakeoffLanding.longitude, lonmin, lonmax))) \ - .filter(TakeoffLanding.altitude < max_altitude) \ + .filter(func.ST_DFullyWithin(TakeoffLanding.location_wkt, Airport.location_wkt, delta_radius)) \ + .filter(TakeoffLanding.altitude < Airport.altitude + delta_altitude) \ .subquery() # find complete flights (with takeoff and landing) with duration < 1 day diff --git a/ogn/gateway/process.py b/ogn/gateway/process.py index 690b3b9..61828d0 100644 --- a/ogn/gateway/process.py +++ b/ogn/gateway/process.py @@ -1,11 +1,19 @@ import logging from ogn.commands.dbutils import session -from ogn.model import AircraftBeacon, ReceiverBeacon +from ogn.model import AircraftBeacon, ReceiverBeacon, Location from ogn.parser import parse_aprs, parse_ogn_receiver_beacon, parse_ogn_aircraft_beacon, ParseError logger = logging.getLogger(__name__) +def replace_lonlat_with_wkt(message): + location = Location(message['longitude'], message['latitude']) + message['location_wkt'] = location.to_wkt() + del message['latitude'] + del message['longitude'] + return message + + def process_beacon(raw_message): if raw_message[0] == '#': return @@ -25,9 +33,11 @@ def process_beacon(raw_message): # /o: ? if message['symboltable'] == "I" and message['symbolcode'] == '&': message.update(parse_ogn_receiver_beacon(message['comment'])) + message = replace_lonlat_with_wkt(message) beacon = ReceiverBeacon(**message) else: message.update(parse_ogn_aircraft_beacon(message['comment'])) + message = replace_lonlat_with_wkt(message) beacon = AircraftBeacon(**message) session.add(beacon) session.commit() @@ -35,3 +45,5 @@ def process_beacon(raw_message): except ParseError as e: logger.error('Received message: {}'.format(raw_message)) logger.error('Drop packet, {}'.format(e.message)) + except TypeError as e: + logger.error('TypeError: {}'.format(raw_message)) diff --git a/ogn/model/__init__.py b/ogn/model/__init__.py index e5fd495..8ff0c22 100644 --- a/ogn/model/__init__.py +++ b/ogn/model/__init__.py @@ -9,3 +9,5 @@ from .receiver_beacon import ReceiverBeacon from .receiver import Receiver from .takeoff_landing import TakeoffLanding from .airport import Airport + +from .geo import Location diff --git a/ogn/model/airport.py b/ogn/model/airport.py index cb3f1e2..16730a4 100644 --- a/ogn/model/airport.py +++ b/ogn/model/airport.py @@ -1,4 +1,5 @@ from sqlalchemy import Column, String, Integer, Float, SmallInteger +from geoalchemy2.types import Geometry from .base import Base @@ -8,14 +9,14 @@ class Airport(Base): id = Column(Integer, primary_key=True) + location_wkt = Column('location', Geometry('POINT', srid=4326)) + altitude = Column(Integer) + name = Column(String, index=True) code = Column(String(5)) country_code = Column(String(2)) style = Column(SmallInteger) description = Column(String) - latitude = Column(Float) - longitude = Column(Float) - altitude = Column(Integer) runway_direction = Column(Integer) runway_length = Column(Integer) frequency = Column(Float) diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index 8a7503d..168a5ea 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -1,21 +1,32 @@ from sqlalchemy import Column, String, Integer, Float, DateTime from sqlalchemy.ext.declarative import AbstractConcreteBase +from geoalchemy2.types import Geometry +from geoalchemy2.shape import to_shape from .base import Base +from .geo import Location class Beacon(AbstractConcreteBase, Base): id = Column(Integer, primary_key=True) # APRS data + location_wkt = Column('location', Geometry('POINT', srid=4326)) + altitude = Column(Integer) + name = Column(String) receiver_name = Column(String(9)) timestamp = Column(DateTime, index=True) - latitude = Column(Float) symboltable = None - longitude = Column(Float) symbolcode = None track = Column(Integer) ground_speed = Column(Float) - altitude = Column(Integer) comment = None + + @property + def location(self): + if self.location_wkt is None: + return None + + coords = to_shape(self.location_wkt) + return Location(lat=coords.y, lon=coords.x) diff --git a/ogn/model/geo.py b/ogn/model/geo.py new file mode 100644 index 0000000..649fc7d --- /dev/null +++ b/ogn/model/geo.py @@ -0,0 +1,15 @@ +class Location: + """Represents a location in WGS84""" + + def __init__(self, lon, lat): + self.longitude = lon + self.latitude = lat + + def to_wkt(self): + return 'SRID=4326;POINT({0} {1})'.format(self.longitude, self.latitude) + + def __str__(self): + return '{0: 7.4f}, {1:8.4f}'.format(self.latitude, self.longitude) + + def as_dict(self): + return {'latitude': round(self.latitude, 8), 'longitude': round(self.longitude, 8)} diff --git a/ogn/model/receiver.py b/ogn/model/receiver.py index 5683a43..fb29a47 100644 --- a/ogn/model/receiver.py +++ b/ogn/model/receiver.py @@ -1,18 +1,30 @@ -from sqlalchemy import Column, String, Integer, Float, DateTime +from sqlalchemy import Column, String, Integer, DateTime +from geoalchemy2.types import Geometry +from geoalchemy2.shape import to_shape from .base import Base +from .geo import Location class Receiver(Base): __tablename__ = "receiver" id = Column(Integer, primary_key=True) - name = Column(String(9)) - latitude = Column(Float) - longitude = Column(Float) + + location_wkt = Column('location', Geometry('POINT', srid=4326)) altitude = Column(Integer) + + name = Column(String(9)) firstseen = Column(DateTime, index=True) lastseen = Column(DateTime, index=True) country_code = Column(String(2)) version = Column(String) platform = Column(String) + + @property + def location(self): + if self.location_wkt is None: + return None + + coords = to_shape(self.location_wkt) + return Location(lat=coords.y, lon=coords.x) diff --git a/ogn/utils.py b/ogn/utils.py index 0741502..d4e352b 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -2,7 +2,7 @@ import requests import csv from io import StringIO -from .model import Device, AddressOrigin, Airport +from .model import Device, AddressOrigin, Airport, Location from geopy.geocoders import Nominatim from geopy.exc import GeopyError @@ -83,22 +83,22 @@ def get_airports(cupfile): airport.country_code = waypoint['country'] airport.style = waypoint['style'] airport.description = waypoint['description'] - airport.latitude = waypoint['latitude'] - airport.longitude = waypoint['longitude'] + location = Location(waypoint['longitude'], waypoint['latitude']) + airport.location_wkt = location.to_wkt() airport.altitude = waypoint['elevation']['value'] if (waypoint['elevation']['unit'] == 'ft'): - airport.altitude = airport.altitude*feet2m + airport.altitude = airport.altitude * feet2m airport.runway_direction = waypoint['runway_direction'] airport.runway_length = waypoint['runway_length']['value'] if (waypoint['runway_length']['unit'] == 'nm'): - airport.altitude = airport.altitude*nm2m + airport.altitude = airport.altitude * nm2m elif (waypoint['runway_length']['unit'] == 'ml'): - airport.altitude = airport.altitude*mi2m + airport.altitude = airport.altitude * mi2m airport.frequency = waypoint['frequency'] airports.append(airport) - except Exception: - print('Failed to parse line: {}'.format(line)) + except AttributeError as e: + print('Failed to parse line: {} {}'.format(line, e)) return airports @@ -115,4 +115,3 @@ def haversine_distance(location0, location1): phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1))) return distance, phi - diff --git a/setup.py b/setup.py index b4fa439..7f9e911 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,8 @@ setup( 'celery[redis]>=3.1,<3.2', 'alembic==0.8.3', 'aerofiles==0.3', + 'geoalchemy2==0.3.0', + 'shapely==1.5.15', 'ogn-client==0.3.0' ], extras_require={