From lat/lon to POINT

Flake8 suggestions

Add shapely

Fixed update_receivers

migrate table takeoff_landing
pull/50/head
Konstantin Gründger 2016-04-24 19:34:25 +02:00
rodzic 201c41f12f
commit 84cb2d264f
12 zmienionych plików z 180 dodań i 56 usunięć

Wyświetl plik

@ -0,0 +1,85 @@
"""Migrate to PostGIS
Revision ID: 277aca1b810
Revises: 3a0765c9a2
Create Date: 2016-04-23 08:01:49.059187
"""
# revision identifiers, used by Alembic.
revision = '277aca1b810'
down_revision = '3a0765c9a2'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
import geoalchemy2 as ga
UPGRADE_QUERY = """
UPDATE {table_name}
SET
location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326);
"""
DOWNGRADE_QUERY = """
UPDATE {table_name}
SET
latitude = ST_Y(ST_TRANSFORM(location, 4326)),
longitude = ST_X(ST_TRANSFORM(location, 4326));
"""
def upgrade():
#CREATE EXTENSION IF NOT EXISTS postgis
op.add_column('airport', sa.Column('location', ga.Geometry('POINT', srid=4326)))
op.execute(UPGRADE_QUERY.format(table_name='airport'))
op.drop_column('airport', 'latitude')
op.drop_column('airport', 'longitude')
op.add_column('aircraft_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
op.execute(UPGRADE_QUERY.format(table_name='aircraft_beacon'))
op.drop_column('aircraft_beacon', 'latitude')
op.drop_column('aircraft_beacon', 'longitude')
op.add_column('receiver_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
op.execute(UPGRADE_QUERY.format(table_name='receiver_beacon'))
op.drop_column('receiver_beacon', 'latitude')
op.drop_column('receiver_beacon', 'longitude')
op.add_column('receiver', sa.Column('location', ga.Geometry('POINT', srid=4326)))
op.execute(UPGRADE_QUERY.format(table_name='receiver'))
op.drop_column('receiver', 'latitude')
op.drop_column('receiver', 'longitude')
op.add_column('takeoff_landing', sa.Column('location', ga.Geometry('POINT', srid=4326)))
op.execute(UPGRADE_QUERY.format(table_name='takeoff_landing'))
op.drop_column('takeoff_landing', 'latitude')
op.drop_column('takeoff_landing', 'longitude')
def downgrade():
#DROP EXTENSION postgis
op.add_column('airport', sa.Column('latitude', sa.FLOAT))
op.add_column('airport', sa.Column('longitude', sa.FLOAT))
op.execute(DOWNGRADE_QUERY.format(table_name='airport'))
op.drop_column('airport', 'location')
op.add_column('aircraft_beacon', sa.Column('latitude', sa.FLOAT))
op.add_column('aircraft_beacon', sa.Column('longitude', sa.FLOAT))
op.execute(DOWNGRADE_QUERY.format(table_name='aircraft_beacon'))
op.drop_column('aircraft_beacon', 'location')
op.add_column('receiver_beacon', sa.Column('latitude', sa.FLOAT))
op.add_column('receiver_beacon', sa.Column('longitude', sa.FLOAT))
op.execute(DOWNGRADE_QUERY.format(table_name='receiver_beacon'))
op.drop_column('receiver_beacon', 'location')
op.add_column('receiver', sa.Column('latitude', sa.FLOAT))
op.add_column('receiver', sa.Column('longitude', sa.FLOAT))
op.execute(DOWNGRADE_QUERY.format(table_name='receiver'))
op.drop_column('receiver', 'location')
op.add_column('takeoff_landing', sa.Column('latitude', sa.FLOAT))
op.add_column('takeoff_landing', sa.Column('longitude', sa.FLOAT))
op.execute(DOWNGRADE_QUERY.format(table_name='takeoff_landing'))
op.drop_column('takeoff_landing', 'location')

Wyświetl plik

@ -41,13 +41,9 @@ def compute_takeoff_and_landing():
AircraftBeacon.timestamp,
func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
AircraftBeacon.latitude,
func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
AircraftBeacon.longitude,
func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
AircraftBeacon.ground_speed,
AircraftBeacon.location_wkt,
func.lag(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_prev'),
func.lead(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_next'),
AircraftBeacon.track,
func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
@ -65,8 +61,7 @@ def compute_takeoff_and_landing():
takeoff_landing_query = app.session.query(
sq.c.address,
sq.c.timestamp,
sq.c.latitude,
sq.c.longitude,
sq.c.location,
sq.c.track,
sq.c.ground_speed,
sq.c.altitude,
@ -82,7 +77,7 @@ def compute_takeoff_and_landing():
.order_by(func.date(sq.c.timestamp), sq.c.timestamp)
# ... and save them
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.location_wkt, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
result = app.session.execute(ins)
counter = result.rowcount
app.session.commit()

Wyświetl plik

@ -1,6 +1,6 @@
from sqlalchemy.sql import func, null
from sqlalchemy.sql.functions import coalesce
from sqlalchemy import and_, or_
from sqlalchemy import and_, not_
from celery.utils.log import get_task_logger
@ -26,8 +26,7 @@ def update_receivers():
.subquery()
receivers_to_update = app.session.query(ReceiverBeacon.name,
ReceiverBeacon.latitude,
ReceiverBeacon.longitude,
ReceiverBeacon.location_wkt,
ReceiverBeacon.altitude,
last_receiver_beacon_sq.columns.lastseen,
ReceiverBeacon.version,
@ -39,11 +38,10 @@ def update_receivers():
# set country code to None if lat or lon changed
count = app.session.query(Receiver) \
.filter(and_(Receiver.name == receivers_to_update.columns.name,
or_(Receiver.latitude != receivers_to_update.columns.latitude,
Receiver.longitude != receivers_to_update.columns.longitude))) \
.update({"latitude": receivers_to_update.columns.latitude,
"longitude": receivers_to_update.columns.longitude,
"country_code": null()})
not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)))) \
.update({"location_wkt": receivers_to_update.columns.location,
"country_code": null()},
synchronize_session=False)
logger.info("Count of receivers who changed lat or lon: {}".format(count))
@ -59,8 +57,7 @@ def update_receivers():
# add new receivers
empty_sq = app.session.query(ReceiverBeacon.name,
ReceiverBeacon.latitude,
ReceiverBeacon.longitude,
ReceiverBeacon.location_wkt,
ReceiverBeacon.altitude,
last_receiver_beacon_sq.columns.lastseen,
ReceiverBeacon.version, ReceiverBeacon.platform) \
@ -73,8 +70,7 @@ def update_receivers():
for receiver_beacon in empty_sq.all():
receiver = Receiver()
receiver.name = receiver_beacon.name
receiver.latitude = receiver_beacon.latitude
receiver.longitude = receiver_beacon.longitude
receiver.location_wkt = receiver_beacon.location_wkt
receiver.altitude = receiver_beacon.altitude
receiver.firstseen = None
receiver.lastseen = receiver_beacon.lastseen
@ -103,7 +99,8 @@ def update_receivers():
.order_by(Receiver.name)
for receiver in unknown_country_query.all():
receiver.country_code = get_country_code(receiver.latitude, receiver.longitude)
location = receiver.location
receiver.country_code = get_country_code(location.latitude, location.longitude)
if receiver.country_code is not None:
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))

Wyświetl plik

@ -3,7 +3,7 @@
from datetime import timedelta
from sqlalchemy.sql import func, null
from sqlalchemy import and_, or_, between
from sqlalchemy import and_, or_
from sqlalchemy.sql.expression import true, false, label
from ogn.model import Device, TakeoffLanding, Airport
@ -28,21 +28,15 @@ def compute():
def show(airport_name):
"""Show a logbook for <airport_name>."""
airport = session.query(Airport) \
.filter(or_(Airport.name==airport_name)) \
.filter(Airport.name == airport_name) \
.first()
if (airport is None):
print('Airport "{}" not found.'.format(airport_name))
return
latitude = float(airport.latitude)
longitude = float(airport.longitude)
altitude = float(airport.altitude)
latmin = latitude - 0.05
latmax = latitude + 0.05
lonmin = longitude - 0.05
lonmax = longitude + 0.05
max_altitude = altitude + 200
delta_altitude = 200
delta_radius = 10
# make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
sq = session.query(
@ -91,9 +85,8 @@ def show(airport_name):
TakeoffLanding.address,
TakeoffLanding.timestamp))
.label('is_takeoff_next')) \
.filter(and_(between(TakeoffLanding.latitude, latmin, latmax),
between(TakeoffLanding.longitude, lonmin, lonmax))) \
.filter(TakeoffLanding.altitude < max_altitude) \
.filter(func.ST_DFullyWithin(TakeoffLanding.location_wkt, Airport.location_wkt, delta_radius)) \
.filter(TakeoffLanding.altitude < Airport.altitude + delta_altitude) \
.subquery()
# find complete flights (with takeoff and landing) with duration < 1 day

Wyświetl plik

@ -1,11 +1,19 @@
import logging
from ogn.commands.dbutils import session
from ogn.model import AircraftBeacon, ReceiverBeacon
from ogn.model import AircraftBeacon, ReceiverBeacon, Location
from ogn.parser import parse_aprs, parse_ogn_receiver_beacon, parse_ogn_aircraft_beacon, ParseError
logger = logging.getLogger(__name__)
def replace_lonlat_with_wkt(message):
location = Location(message['longitude'], message['latitude'])
message['location_wkt'] = location.to_wkt()
del message['latitude']
del message['longitude']
return message
def process_beacon(raw_message):
if raw_message[0] == '#':
return
@ -25,9 +33,11 @@ def process_beacon(raw_message):
# /o: ?
if message['symboltable'] == "I" and message['symbolcode'] == '&':
message.update(parse_ogn_receiver_beacon(message['comment']))
message = replace_lonlat_with_wkt(message)
beacon = ReceiverBeacon(**message)
else:
message.update(parse_ogn_aircraft_beacon(message['comment']))
message = replace_lonlat_with_wkt(message)
beacon = AircraftBeacon(**message)
session.add(beacon)
session.commit()
@ -35,3 +45,5 @@ def process_beacon(raw_message):
except ParseError as e:
logger.error('Received message: {}'.format(raw_message))
logger.error('Drop packet, {}'.format(e.message))
except TypeError as e:
logger.error('TypeError: {}'.format(raw_message))

Wyświetl plik

@ -9,3 +9,5 @@ from .receiver_beacon import ReceiverBeacon
from .receiver import Receiver
from .takeoff_landing import TakeoffLanding
from .airport import Airport
from .geo import Location

Wyświetl plik

@ -1,4 +1,5 @@
from sqlalchemy import Column, String, Integer, Float, SmallInteger
from geoalchemy2.types import Geometry
from .base import Base
@ -8,14 +9,14 @@ class Airport(Base):
id = Column(Integer, primary_key=True)
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Integer)
name = Column(String, index=True)
code = Column(String(5))
country_code = Column(String(2))
style = Column(SmallInteger)
description = Column(String)
latitude = Column(Float)
longitude = Column(Float)
altitude = Column(Integer)
runway_direction = Column(Integer)
runway_length = Column(Integer)
frequency = Column(Float)

Wyświetl plik

@ -1,21 +1,32 @@
from sqlalchemy import Column, String, Integer, Float, DateTime
from sqlalchemy.ext.declarative import AbstractConcreteBase
from geoalchemy2.types import Geometry
from geoalchemy2.shape import to_shape
from .base import Base
from .geo import Location
class Beacon(AbstractConcreteBase, Base):
id = Column(Integer, primary_key=True)
# APRS data
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Integer)
name = Column(String)
receiver_name = Column(String(9))
timestamp = Column(DateTime, index=True)
latitude = Column(Float)
symboltable = None
longitude = Column(Float)
symbolcode = None
track = Column(Integer)
ground_speed = Column(Float)
altitude = Column(Integer)
comment = None
@property
def location(self):
if self.location_wkt is None:
return None
coords = to_shape(self.location_wkt)
return Location(lat=coords.y, lon=coords.x)

15
ogn/model/geo.py 100644
Wyświetl plik

@ -0,0 +1,15 @@
class Location:
"""Represents a location in WGS84"""
def __init__(self, lon, lat):
self.longitude = lon
self.latitude = lat
def to_wkt(self):
return 'SRID=4326;POINT({0} {1})'.format(self.longitude, self.latitude)
def __str__(self):
return '{0: 7.4f}, {1:8.4f}'.format(self.latitude, self.longitude)
def as_dict(self):
return {'latitude': round(self.latitude, 8), 'longitude': round(self.longitude, 8)}

Wyświetl plik

@ -1,18 +1,30 @@
from sqlalchemy import Column, String, Integer, Float, DateTime
from sqlalchemy import Column, String, Integer, DateTime
from geoalchemy2.types import Geometry
from geoalchemy2.shape import to_shape
from .base import Base
from .geo import Location
class Receiver(Base):
__tablename__ = "receiver"
id = Column(Integer, primary_key=True)
name = Column(String(9))
latitude = Column(Float)
longitude = Column(Float)
location_wkt = Column('location', Geometry('POINT', srid=4326))
altitude = Column(Integer)
name = Column(String(9))
firstseen = Column(DateTime, index=True)
lastseen = Column(DateTime, index=True)
country_code = Column(String(2))
version = Column(String)
platform = Column(String)
@property
def location(self):
if self.location_wkt is None:
return None
coords = to_shape(self.location_wkt)
return Location(lat=coords.y, lon=coords.x)

Wyświetl plik

@ -2,7 +2,7 @@ import requests
import csv
from io import StringIO
from .model import Device, AddressOrigin, Airport
from .model import Device, AddressOrigin, Airport, Location
from geopy.geocoders import Nominatim
from geopy.exc import GeopyError
@ -83,22 +83,22 @@ def get_airports(cupfile):
airport.country_code = waypoint['country']
airport.style = waypoint['style']
airport.description = waypoint['description']
airport.latitude = waypoint['latitude']
airport.longitude = waypoint['longitude']
location = Location(waypoint['longitude'], waypoint['latitude'])
airport.location_wkt = location.to_wkt()
airport.altitude = waypoint['elevation']['value']
if (waypoint['elevation']['unit'] == 'ft'):
airport.altitude = airport.altitude*feet2m
airport.altitude = airport.altitude * feet2m
airport.runway_direction = waypoint['runway_direction']
airport.runway_length = waypoint['runway_length']['value']
if (waypoint['runway_length']['unit'] == 'nm'):
airport.altitude = airport.altitude*nm2m
airport.altitude = airport.altitude * nm2m
elif (waypoint['runway_length']['unit'] == 'ml'):
airport.altitude = airport.altitude*mi2m
airport.altitude = airport.altitude * mi2m
airport.frequency = waypoint['frequency']
airports.append(airport)
except Exception:
print('Failed to parse line: {}'.format(line))
except AttributeError as e:
print('Failed to parse line: {} {}'.format(line, e))
return airports
@ -115,4 +115,3 @@ def haversine_distance(location0, location1):
phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1)))
return distance, phi

Wyświetl plik

@ -38,6 +38,8 @@ setup(
'celery[redis]>=3.1,<3.2',
'alembic==0.8.3',
'aerofiles==0.3',
'geoalchemy2==0.3.0',
'shapely==1.5.15',
'ogn-client==0.3.0'
],
extras_require={