diff --git a/ogn/collect/database.py b/ogn/collect/database.py index 071a53c..479d1f8 100644 --- a/ogn/collect/database.py +++ b/ogn/collect/database.py @@ -5,8 +5,8 @@ from sqlalchemy.sql import null, and_, or_, func, not_ from sqlalchemy.sql.expression import case from ogn.collect.celery import app -from ogn.model import DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver -from ogn.utils import get_ddb, get_country_code, get_flarmnet +from ogn.model import Country, DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn.utils import get_ddb, get_flarmnet logger = get_task_logger(__name__) @@ -130,21 +130,12 @@ def update_country_code(session=None): if session is None: session = app.session - unknown_country_query = session.query(Receiver) \ - .filter(Receiver.country_code == null()) \ - .filter(Receiver.location_wkt != null()) \ - .order_by(Receiver.name) - - counter = 0 - for receiver in unknown_country_query.all(): - location = receiver.location - country_code = get_country_code(location.latitude, location.longitude) - print("{}: {}".format(receiver.name, country_code)) - if country_code is not None: - receiver.country_code = country_code - logger.info("Updated country_code for {} to {}".format(receiver.name, receiver.country_code)) - counter += 1 + update_receivers = session.query(Receiver) \ + .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom))) \ + .update({Receiver.country_id: Country.gid}, + synchronize_session='fetch') session.commit() + logger.info("Updated {} AircraftBeacons".format(update_receivers)) - return "Updated country_code for {} Receivers".format(counter) + return "Updated country for {} Receivers".format(update_receivers) diff --git a/ogn/model/__init__.py b/ogn/model/__init__.py index 5e6b754..f302afc 100644 --- a/ogn/model/__init__.py +++ b/ogn/model/__init__.py @@ -2,6 +2,7 @@ from .aircraft_type import AircraftType from .base import Base from .beacon import Beacon +from .country import Country from .device import Device from .device_info import DeviceInfo from .device_info_origin import DeviceInfoOrigin diff --git a/ogn/model/country.py b/ogn/model/country.py new file mode 100644 index 0000000..062f0e5 --- /dev/null +++ b/ogn/model/country.py @@ -0,0 +1,40 @@ +from geoalchemy2.types import Geometry +from sqlalchemy import Column, String, Integer, Float, SmallInteger, BigInteger +from sqlalchemy.orm import relationship + +from .base import Base + + +class Country(Base): + __tablename__ = "countries" + + gid = Column(Integer, primary_key=True) + + fips = Column(String(2)) + iso2 = Column(String(2)) + iso3 = Column(String(3)) + + un = Column(SmallInteger) + name = Column(String(50)) + area = Column(Integer) + pop2005 = Column(BigInteger) + region = Column(SmallInteger) + subregion = Column(SmallInteger) + lon = Column(Float) + lat = Column(Float) + + geom = Column('geom', Geometry('MULTIPOLYGON', srid=4326)) + + def __repr__(self): + return "" % ( + self.name, + self.code, + self.country_code, + self.style, + self.description, + self.location_wkt.latitude if self.location_wkt else None, + self.location_wkt.longitude if self.location_wkt else None, + self.altitude, + self.runway_direction, + self.runway_length, + self.frequency) diff --git a/ogn/model/receiver.py b/ogn/model/receiver.py index baa94a0..51b5552 100644 --- a/ogn/model/receiver.py +++ b/ogn/model/receiver.py @@ -1,7 +1,7 @@ from geoalchemy2.shape import to_shape from geoalchemy2.types import Geometry -from sqlalchemy import Column, Float, String, Integer, DateTime -from sqlalchemy.orm import relationship +from sqlalchemy import Column, Float, String, Integer, DateTime, ForeignKey +from sqlalchemy.orm import relationship, backref from .base import Base from .geo import Location @@ -18,10 +18,13 @@ class Receiver(Base): name = Column(String(9), index=True) firstseen = Column(DateTime, index=True) lastseen = Column(DateTime, index=True) - country_code = Column(String(2), index=True) version = Column(String) platform = Column(String) + # Relations + country_id = Column(Integer, ForeignKey('countries.gid', ondelete='SET NULL'), index=True) + country = relationship('Country', foreign_keys=[country_id], backref=backref('receivers', order_by='Receiver.name.asc()')) + @property def location(self): if self.location_wkt is None: diff --git a/ogn/utils.py b/ogn/utils.py index f68bb06..3a39b4c 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -81,33 +81,6 @@ def get_trackable(ddb): return l -def get_geolocator(): - geolocator = Nominatim() - - requester = geolocator.urlopen - - def requester_hack(req, **kwargs): - req = Request(url=req, headers=geolocator.headers) - return requester(req, **kwargs) - - geolocator.urlopen = requester_hack - - return geolocator - - -def get_country_code(latitude, longitude): - geolocator = get_geolocator() - try: - location = geolocator.reverse("{}, {}".format(latitude, longitude)) - country_code = location.raw['address']['country_code'] - except KeyError as e: - country_code = None - except GeopyError as e: - print(e) - country_code = None - return country_code - - def get_airports(cupfile): airports = list() with open(cupfile) as f: diff --git a/tests/test_utils.py b/tests/test_utils.py index 8bb2a2f..a5fa8be 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -33,27 +33,6 @@ class TestStringMethods(unittest.TestCase): self.assertIn('OGNDEADBE', trackable) self.assertIn('ICA999999', trackable) - def test_get_country_code(self): - latitude = 48.0 - longitude = 11.0 - country_code = get_country_code(latitude, longitude) - self.assertEquals(country_code, 'de') - - def test_get_country_code_bad(self): - latitude = 0.0002274 - longitude = -0.0009119 - country_code = get_country_code(latitude, longitude) - self.assertEqual(country_code, None) - - @mock.patch('ogn.utils.Nominatim') - def test_gec_country_code_exception(self, nominatim_mock): - from geopy.exc import GeocoderTimedOut - instance = nominatim_mock.return_value - - instance.reverse.side_effect = GeocoderTimedOut('Too busy') - country_code = get_country_code(0, 0) - self.assertIsNone(country_code) - def test_get_airports(self): airports = get_airports(os.path.dirname(__file__) + '/SeeYou.cup') self.assertGreater(len(airports), 1000)