kopia lustrzana https://github.com/glidernet/ogn-python
From lat/lon to POINT
Flake8 suggestions Add shapely Fixed update_receivers migrate table takeoff_landingpull/50/head
rodzic
201c41f12f
commit
84cb2d264f
|
@ -0,0 +1,85 @@
|
|||
"""Migrate to PostGIS
|
||||
|
||||
Revision ID: 277aca1b810
|
||||
Revises: 3a0765c9a2
|
||||
Create Date: 2016-04-23 08:01:49.059187
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '277aca1b810'
|
||||
down_revision = '3a0765c9a2'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
import geoalchemy2 as ga
|
||||
|
||||
UPGRADE_QUERY = """
|
||||
UPDATE {table_name}
|
||||
SET
|
||||
location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326);
|
||||
"""
|
||||
|
||||
DOWNGRADE_QUERY = """
|
||||
UPDATE {table_name}
|
||||
SET
|
||||
latitude = ST_Y(ST_TRANSFORM(location, 4326)),
|
||||
longitude = ST_X(ST_TRANSFORM(location, 4326));
|
||||
"""
|
||||
|
||||
def upgrade():
|
||||
#CREATE EXTENSION IF NOT EXISTS postgis
|
||||
op.add_column('airport', sa.Column('location', ga.Geometry('POINT', srid=4326)))
|
||||
op.execute(UPGRADE_QUERY.format(table_name='airport'))
|
||||
op.drop_column('airport', 'latitude')
|
||||
op.drop_column('airport', 'longitude')
|
||||
|
||||
op.add_column('aircraft_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
|
||||
op.execute(UPGRADE_QUERY.format(table_name='aircraft_beacon'))
|
||||
op.drop_column('aircraft_beacon', 'latitude')
|
||||
op.drop_column('aircraft_beacon', 'longitude')
|
||||
|
||||
op.add_column('receiver_beacon', sa.Column('location', ga.Geometry('POINT', srid=4326)))
|
||||
op.execute(UPGRADE_QUERY.format(table_name='receiver_beacon'))
|
||||
op.drop_column('receiver_beacon', 'latitude')
|
||||
op.drop_column('receiver_beacon', 'longitude')
|
||||
|
||||
op.add_column('receiver', sa.Column('location', ga.Geometry('POINT', srid=4326)))
|
||||
op.execute(UPGRADE_QUERY.format(table_name='receiver'))
|
||||
op.drop_column('receiver', 'latitude')
|
||||
op.drop_column('receiver', 'longitude')
|
||||
|
||||
op.add_column('takeoff_landing', sa.Column('location', ga.Geometry('POINT', srid=4326)))
|
||||
op.execute(UPGRADE_QUERY.format(table_name='takeoff_landing'))
|
||||
op.drop_column('takeoff_landing', 'latitude')
|
||||
op.drop_column('takeoff_landing', 'longitude')
|
||||
|
||||
|
||||
def downgrade():
|
||||
#DROP EXTENSION postgis
|
||||
op.add_column('airport', sa.Column('latitude', sa.FLOAT))
|
||||
op.add_column('airport', sa.Column('longitude', sa.FLOAT))
|
||||
op.execute(DOWNGRADE_QUERY.format(table_name='airport'))
|
||||
op.drop_column('airport', 'location')
|
||||
|
||||
op.add_column('aircraft_beacon', sa.Column('latitude', sa.FLOAT))
|
||||
op.add_column('aircraft_beacon', sa.Column('longitude', sa.FLOAT))
|
||||
op.execute(DOWNGRADE_QUERY.format(table_name='aircraft_beacon'))
|
||||
op.drop_column('aircraft_beacon', 'location')
|
||||
|
||||
op.add_column('receiver_beacon', sa.Column('latitude', sa.FLOAT))
|
||||
op.add_column('receiver_beacon', sa.Column('longitude', sa.FLOAT))
|
||||
op.execute(DOWNGRADE_QUERY.format(table_name='receiver_beacon'))
|
||||
op.drop_column('receiver_beacon', 'location')
|
||||
|
||||
op.add_column('receiver', sa.Column('latitude', sa.FLOAT))
|
||||
op.add_column('receiver', sa.Column('longitude', sa.FLOAT))
|
||||
op.execute(DOWNGRADE_QUERY.format(table_name='receiver'))
|
||||
op.drop_column('receiver', 'location')
|
||||
|
||||
op.add_column('takeoff_landing', sa.Column('latitude', sa.FLOAT))
|
||||
op.add_column('takeoff_landing', sa.Column('longitude', sa.FLOAT))
|
||||
op.execute(DOWNGRADE_QUERY.format(table_name='takeoff_landing'))
|
||||
op.drop_column('takeoff_landing', 'location')
|
|
@ -41,13 +41,9 @@ def compute_takeoff_and_landing():
|
|||
AircraftBeacon.timestamp,
|
||||
func.lag(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_prev'),
|
||||
func.lead(AircraftBeacon.timestamp).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('timestamp_next'),
|
||||
AircraftBeacon.latitude,
|
||||
func.lag(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_prev'),
|
||||
func.lead(AircraftBeacon.latitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('latitude_next'),
|
||||
AircraftBeacon.longitude,
|
||||
func.lag(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_prev'),
|
||||
func.lead(AircraftBeacon.longitude).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('longitude_next'),
|
||||
AircraftBeacon.ground_speed,
|
||||
AircraftBeacon.location_wkt,
|
||||
func.lag(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_prev'),
|
||||
func.lead(AircraftBeacon.location_wkt).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('location_wkt_next'),
|
||||
AircraftBeacon.track,
|
||||
func.lag(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_prev'),
|
||||
func.lead(AircraftBeacon.track).over(order_by=and_(AircraftBeacon.address, AircraftBeacon.timestamp)).label('track_next'),
|
||||
|
@ -65,8 +61,7 @@ def compute_takeoff_and_landing():
|
|||
takeoff_landing_query = app.session.query(
|
||||
sq.c.address,
|
||||
sq.c.timestamp,
|
||||
sq.c.latitude,
|
||||
sq.c.longitude,
|
||||
sq.c.location,
|
||||
sq.c.track,
|
||||
sq.c.ground_speed,
|
||||
sq.c.altitude,
|
||||
|
@ -82,7 +77,7 @@ def compute_takeoff_and_landing():
|
|||
.order_by(func.date(sq.c.timestamp), sq.c.timestamp)
|
||||
|
||||
# ... and save them
|
||||
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.latitude, TakeoffLanding.longitude, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
|
||||
ins = insert(TakeoffLanding).from_select((TakeoffLanding.address, TakeoffLanding.timestamp, TakeoffLanding.location_wkt, TakeoffLanding.track, TakeoffLanding.ground_speed, TakeoffLanding.altitude, TakeoffLanding.is_takeoff), takeoff_landing_query)
|
||||
result = app.session.execute(ins)
|
||||
counter = result.rowcount
|
||||
app.session.commit()
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
from sqlalchemy.sql import func, null
|
||||
from sqlalchemy.sql.functions import coalesce
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlalchemy import and_, not_
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
|
@ -26,8 +26,7 @@ def update_receivers():
|
|||
.subquery()
|
||||
|
||||
receivers_to_update = app.session.query(ReceiverBeacon.name,
|
||||
ReceiverBeacon.latitude,
|
||||
ReceiverBeacon.longitude,
|
||||
ReceiverBeacon.location_wkt,
|
||||
ReceiverBeacon.altitude,
|
||||
last_receiver_beacon_sq.columns.lastseen,
|
||||
ReceiverBeacon.version,
|
||||
|
@ -39,11 +38,10 @@ def update_receivers():
|
|||
# set country code to None if lat or lon changed
|
||||
count = app.session.query(Receiver) \
|
||||
.filter(and_(Receiver.name == receivers_to_update.columns.name,
|
||||
or_(Receiver.latitude != receivers_to_update.columns.latitude,
|
||||
Receiver.longitude != receivers_to_update.columns.longitude))) \
|
||||
.update({"latitude": receivers_to_update.columns.latitude,
|
||||
"longitude": receivers_to_update.columns.longitude,
|
||||
"country_code": null()})
|
||||
not_(func.ST_Equals(Receiver.location_wkt, receivers_to_update.columns.location)))) \
|
||||
.update({"location_wkt": receivers_to_update.columns.location,
|
||||
"country_code": null()},
|
||||
synchronize_session=False)
|
||||
|
||||
logger.info("Count of receivers who changed lat or lon: {}".format(count))
|
||||
|
||||
|
@ -59,8 +57,7 @@ def update_receivers():
|
|||
|
||||
# add new receivers
|
||||
empty_sq = app.session.query(ReceiverBeacon.name,
|
||||
ReceiverBeacon.latitude,
|
||||
ReceiverBeacon.longitude,
|
||||
ReceiverBeacon.location_wkt,
|
||||
ReceiverBeacon.altitude,
|
||||
last_receiver_beacon_sq.columns.lastseen,
|
||||
ReceiverBeacon.version, ReceiverBeacon.platform) \
|
||||
|
@ -73,8 +70,7 @@ def update_receivers():
|
|||
for receiver_beacon in empty_sq.all():
|
||||
receiver = Receiver()
|
||||
receiver.name = receiver_beacon.name
|
||||
receiver.latitude = receiver_beacon.latitude
|
||||
receiver.longitude = receiver_beacon.longitude
|
||||
receiver.location_wkt = receiver_beacon.location_wkt
|
||||
receiver.altitude = receiver_beacon.altitude
|
||||
receiver.firstseen = None
|
||||
receiver.lastseen = receiver_beacon.lastseen
|
||||
|
@ -103,7 +99,8 @@ def update_receivers():
|
|||
.order_by(Receiver.name)
|
||||
|
||||
for receiver in unknown_country_query.all():
|
||||
receiver.country_code = get_country_code(receiver.latitude, receiver.longitude)
|
||||
location = receiver.location
|
||||
receiver.country_code = get_country_code(location.latitude, location.longitude)
|
||||
if receiver.country_code is not None:
|
||||
logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code))
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy.sql import func, null
|
||||
from sqlalchemy import and_, or_, between
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlalchemy.sql.expression import true, false, label
|
||||
|
||||
from ogn.model import Device, TakeoffLanding, Airport
|
||||
|
@ -28,21 +28,15 @@ def compute():
|
|||
def show(airport_name):
|
||||
"""Show a logbook for <airport_name>."""
|
||||
airport = session.query(Airport) \
|
||||
.filter(or_(Airport.name==airport_name)) \
|
||||
.filter(Airport.name == airport_name) \
|
||||
.first()
|
||||
|
||||
if (airport is None):
|
||||
print('Airport "{}" not found.'.format(airport_name))
|
||||
return
|
||||
|
||||
latitude = float(airport.latitude)
|
||||
longitude = float(airport.longitude)
|
||||
altitude = float(airport.altitude)
|
||||
latmin = latitude - 0.05
|
||||
latmax = latitude + 0.05
|
||||
lonmin = longitude - 0.05
|
||||
lonmax = longitude + 0.05
|
||||
max_altitude = altitude + 200
|
||||
delta_altitude = 200
|
||||
delta_radius = 10
|
||||
|
||||
# make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
|
||||
sq = session.query(
|
||||
|
@ -91,9 +85,8 @@ def show(airport_name):
|
|||
TakeoffLanding.address,
|
||||
TakeoffLanding.timestamp))
|
||||
.label('is_takeoff_next')) \
|
||||
.filter(and_(between(TakeoffLanding.latitude, latmin, latmax),
|
||||
between(TakeoffLanding.longitude, lonmin, lonmax))) \
|
||||
.filter(TakeoffLanding.altitude < max_altitude) \
|
||||
.filter(func.ST_DFullyWithin(TakeoffLanding.location_wkt, Airport.location_wkt, delta_radius)) \
|
||||
.filter(TakeoffLanding.altitude < Airport.altitude + delta_altitude) \
|
||||
.subquery()
|
||||
|
||||
# find complete flights (with takeoff and landing) with duration < 1 day
|
||||
|
|
|
@ -1,11 +1,19 @@
|
|||
import logging
|
||||
from ogn.commands.dbutils import session
|
||||
from ogn.model import AircraftBeacon, ReceiverBeacon
|
||||
from ogn.model import AircraftBeacon, ReceiverBeacon, Location
|
||||
from ogn.parser import parse_aprs, parse_ogn_receiver_beacon, parse_ogn_aircraft_beacon, ParseError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def replace_lonlat_with_wkt(message):
|
||||
location = Location(message['longitude'], message['latitude'])
|
||||
message['location_wkt'] = location.to_wkt()
|
||||
del message['latitude']
|
||||
del message['longitude']
|
||||
return message
|
||||
|
||||
|
||||
def process_beacon(raw_message):
|
||||
if raw_message[0] == '#':
|
||||
return
|
||||
|
@ -25,9 +33,11 @@ def process_beacon(raw_message):
|
|||
# /o: ?
|
||||
if message['symboltable'] == "I" and message['symbolcode'] == '&':
|
||||
message.update(parse_ogn_receiver_beacon(message['comment']))
|
||||
message = replace_lonlat_with_wkt(message)
|
||||
beacon = ReceiverBeacon(**message)
|
||||
else:
|
||||
message.update(parse_ogn_aircraft_beacon(message['comment']))
|
||||
message = replace_lonlat_with_wkt(message)
|
||||
beacon = AircraftBeacon(**message)
|
||||
session.add(beacon)
|
||||
session.commit()
|
||||
|
@ -35,3 +45,5 @@ def process_beacon(raw_message):
|
|||
except ParseError as e:
|
||||
logger.error('Received message: {}'.format(raw_message))
|
||||
logger.error('Drop packet, {}'.format(e.message))
|
||||
except TypeError as e:
|
||||
logger.error('TypeError: {}'.format(raw_message))
|
||||
|
|
|
@ -9,3 +9,5 @@ from .receiver_beacon import ReceiverBeacon
|
|||
from .receiver import Receiver
|
||||
from .takeoff_landing import TakeoffLanding
|
||||
from .airport import Airport
|
||||
|
||||
from .geo import Location
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
from sqlalchemy import Column, String, Integer, Float, SmallInteger
|
||||
from geoalchemy2.types import Geometry
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
@ -8,14 +9,14 @@ class Airport(Base):
|
|||
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Integer)
|
||||
|
||||
name = Column(String, index=True)
|
||||
code = Column(String(5))
|
||||
country_code = Column(String(2))
|
||||
style = Column(SmallInteger)
|
||||
description = Column(String)
|
||||
latitude = Column(Float)
|
||||
longitude = Column(Float)
|
||||
altitude = Column(Integer)
|
||||
runway_direction = Column(Integer)
|
||||
runway_length = Column(Integer)
|
||||
frequency = Column(Float)
|
||||
|
|
|
@ -1,21 +1,32 @@
|
|||
from sqlalchemy import Column, String, Integer, Float, DateTime
|
||||
from sqlalchemy.ext.declarative import AbstractConcreteBase
|
||||
from geoalchemy2.types import Geometry
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from .base import Base
|
||||
from .geo import Location
|
||||
|
||||
|
||||
class Beacon(AbstractConcreteBase, Base):
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# APRS data
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Integer)
|
||||
|
||||
name = Column(String)
|
||||
receiver_name = Column(String(9))
|
||||
timestamp = Column(DateTime, index=True)
|
||||
latitude = Column(Float)
|
||||
symboltable = None
|
||||
longitude = Column(Float)
|
||||
symbolcode = None
|
||||
track = Column(Integer)
|
||||
ground_speed = Column(Float)
|
||||
altitude = Column(Integer)
|
||||
comment = None
|
||||
|
||||
@property
|
||||
def location(self):
|
||||
if self.location_wkt is None:
|
||||
return None
|
||||
|
||||
coords = to_shape(self.location_wkt)
|
||||
return Location(lat=coords.y, lon=coords.x)
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
class Location:
|
||||
"""Represents a location in WGS84"""
|
||||
|
||||
def __init__(self, lon, lat):
|
||||
self.longitude = lon
|
||||
self.latitude = lat
|
||||
|
||||
def to_wkt(self):
|
||||
return 'SRID=4326;POINT({0} {1})'.format(self.longitude, self.latitude)
|
||||
|
||||
def __str__(self):
|
||||
return '{0: 7.4f}, {1:8.4f}'.format(self.latitude, self.longitude)
|
||||
|
||||
def as_dict(self):
|
||||
return {'latitude': round(self.latitude, 8), 'longitude': round(self.longitude, 8)}
|
|
@ -1,18 +1,30 @@
|
|||
from sqlalchemy import Column, String, Integer, Float, DateTime
|
||||
from sqlalchemy import Column, String, Integer, DateTime
|
||||
from geoalchemy2.types import Geometry
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from .base import Base
|
||||
from .geo import Location
|
||||
|
||||
|
||||
class Receiver(Base):
|
||||
__tablename__ = "receiver"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String(9))
|
||||
latitude = Column(Float)
|
||||
longitude = Column(Float)
|
||||
|
||||
location_wkt = Column('location', Geometry('POINT', srid=4326))
|
||||
altitude = Column(Integer)
|
||||
|
||||
name = Column(String(9))
|
||||
firstseen = Column(DateTime, index=True)
|
||||
lastseen = Column(DateTime, index=True)
|
||||
country_code = Column(String(2))
|
||||
version = Column(String)
|
||||
platform = Column(String)
|
||||
|
||||
@property
|
||||
def location(self):
|
||||
if self.location_wkt is None:
|
||||
return None
|
||||
|
||||
coords = to_shape(self.location_wkt)
|
||||
return Location(lat=coords.y, lon=coords.x)
|
||||
|
|
17
ogn/utils.py
17
ogn/utils.py
|
@ -2,7 +2,7 @@ import requests
|
|||
import csv
|
||||
from io import StringIO
|
||||
|
||||
from .model import Device, AddressOrigin, Airport
|
||||
from .model import Device, AddressOrigin, Airport, Location
|
||||
|
||||
from geopy.geocoders import Nominatim
|
||||
from geopy.exc import GeopyError
|
||||
|
@ -83,22 +83,22 @@ def get_airports(cupfile):
|
|||
airport.country_code = waypoint['country']
|
||||
airport.style = waypoint['style']
|
||||
airport.description = waypoint['description']
|
||||
airport.latitude = waypoint['latitude']
|
||||
airport.longitude = waypoint['longitude']
|
||||
location = Location(waypoint['longitude'], waypoint['latitude'])
|
||||
airport.location_wkt = location.to_wkt()
|
||||
airport.altitude = waypoint['elevation']['value']
|
||||
if (waypoint['elevation']['unit'] == 'ft'):
|
||||
airport.altitude = airport.altitude*feet2m
|
||||
airport.altitude = airport.altitude * feet2m
|
||||
airport.runway_direction = waypoint['runway_direction']
|
||||
airport.runway_length = waypoint['runway_length']['value']
|
||||
if (waypoint['runway_length']['unit'] == 'nm'):
|
||||
airport.altitude = airport.altitude*nm2m
|
||||
airport.altitude = airport.altitude * nm2m
|
||||
elif (waypoint['runway_length']['unit'] == 'ml'):
|
||||
airport.altitude = airport.altitude*mi2m
|
||||
airport.altitude = airport.altitude * mi2m
|
||||
airport.frequency = waypoint['frequency']
|
||||
|
||||
airports.append(airport)
|
||||
except Exception:
|
||||
print('Failed to parse line: {}'.format(line))
|
||||
except AttributeError as e:
|
||||
print('Failed to parse line: {} {}'.format(line, e))
|
||||
|
||||
return airports
|
||||
|
||||
|
@ -115,4 +115,3 @@ def haversine_distance(location0, location1):
|
|||
phi = degrees(atan2(sin(lon0 - lon1) * cos(lat1), cos(lat0) * sin(lat1) - sin(lat0) * cos(lat1) * cos(lon0 - lon1)))
|
||||
|
||||
return distance, phi
|
||||
|
||||
|
|
2
setup.py
2
setup.py
|
@ -38,6 +38,8 @@ setup(
|
|||
'celery[redis]>=3.1,<3.2',
|
||||
'alembic==0.8.3',
|
||||
'aerofiles==0.3',
|
||||
'geoalchemy2==0.3.0',
|
||||
'shapely==1.5.15',
|
||||
'ogn-client==0.3.0'
|
||||
],
|
||||
extras_require={
|
||||
|
|
Ładowanie…
Reference in New Issue