Refactoring WIP

Vagrantfile

@@ -4,7 +4,7 @@ set -e
 
 # install PostgreSQL and PostGIS
 sudo apt-get update
-sudo apt-get install -y --no-install-recommends postgresql-9.4-postgis-2.1 libpq-dev
+sudo apt-get install -y postgresql-11-postgis-2.5
 
 # create PostGIS database
 sudo sudo -u postgres createuser -s vagrant
@@ -14,22 +14,18 @@ sudo sudo -u postgres psql -d ogn -c 'CREATE EXTENSION postgis;'
 
 # install python requirements
 cd /vagrant
-sudo apt-get install -y --no-install-recommends redis-server build-essential python3 python3-pip python3-dev libpq-dev libgeos-dev
-sudo -H pip3 install -r requirements.txt
+sudo apt-get install -y python3-pip redis-server
+sudo pip3 install -r requirements.txt
 
 # # initialize database
-./manage.py db.init
+#./manage.py db.init
 
 # # import registered devices from ddb
-./manage.py db.import_ddb
+#./manage.py db.import_ddb
 
 SCRIPT
 
 Vagrant.configure("2") do |config|
-  config.vm.box = 'debian/jessie64'
-
-  # Current version is broken
-  config.vm.box_version = '8.5.2'
-
+  config.vm.box = 'debian/buster64'
   config.vm.provision 'shell', inline: $script, privileged: false
 end

app/__init__.py

@@ -5,14 +5,14 @@ from flask_migrate import Migrate
 from flask_caching import Cache
 from celery import Celery
 
-from ogn_python.flask_celery import make_celery
+from app.flask_celery import make_celery
 
 # Initialize Flask
 app = Flask(__name__)
 
 # Load the configuration
-#app.config.from_object('config.default')
-app.config.from_envvar('OGN_CONFIG_MODULE')
+app.config.from_object('app.config.default')
+app.config.from_envvar("OGN_CONFIG_MODULE", silent=True)
 
 # Initialize other things
 bootstrap = Bootstrap(app)
@@ -20,3 +20,5 @@ db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 cache = Cache(app)
 celery = make_celery(app)
+
+from app import routes, commands

app/backend/liveglidernet.py

@@ -0,0 +1,105 @@
+from datetime import datetime, timedelta, timezone, date
+
+from app.model import AircraftBeacon, Device, Receiver
+
+from app import db
+from app.model import ReceiverBeacon
+
+
+def utc_to_local(utc_dt):
+    return utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None)
+
+
+def encode(address):
+    return "xx" + address
+
+
+def decode(code):
+    return code[2:9]
+
+
+def rec(min_timestamp, min_online_timestamp):
+    last_seen_query = (
+        db.session.query(ReceiverBeacon).filter(ReceiverBeacon.timestamp > min_timestamp).order_by(ReceiverBeacon.receiver_id, ReceiverBeacon.timestamp).distinct(ReceiverBeacon.receiver_id)
+    )
+
+    lines = []
+    lines.append('<?xml version="1.0" encoding="UTF-8"?>')
+    lines.append("<markers>")
+    lines.append('<m e="0"/>')
+    for receiver_beacon in last_seen_query:
+        if receiver_beacon.location == None or receiver_beacon.name.startswith("FNB"):
+            continue
+        lines.append(
+            '<m a="{0}" b="{1:.7f}" c="{2:.7f}" d="{3:1d}"/>'.format(
+                receiver_beacon.name, receiver_beacon.location.latitude, receiver_beacon.location.longitude, receiver_beacon.timestamp < min_online_timestamp
+            )
+        )
+
+    lines.append("</markers>")
+    xml = "\n".join(lines)
+
+    return xml
+
+
+def lxml(show_offline=False, lat_max=90, lat_min=-90, lon_max=180, lon_min=-180):
+
+    timestamp_range_filter = [db.between(AircraftBeacon.timestamp, datetime(2018, 7, 31, 11, 55, 0), datetime(2018, 7, 31, 12, 5, 0))]
+
+    last_seen_query = db.session.query(AircraftBeacon).filter(*timestamp_range_filter).order_by(AircraftBeacon.device_id, AircraftBeacon.timestamp).distinct(AircraftBeacon.device_id)
+    lines = list()
+    lines.append('<?xml version="1.0" encoding="UTF-8"?>')
+    lines.append("<markers>")
+
+    for aircraft_beacon in last_seen_query:
+        device = aircraft_beacon.device
+
+        code = encode(device.address)
+
+        if device.info:
+            if not device.info.tracked or not device.info.identified:
+                continue
+
+            if not device.info.competition:
+                competition = device.info.registration[-2:]
+            else:
+                competition = device.info.competition
+
+            if not device.info.registration:
+                registration = "???"
+            else:
+                registration = device.info.registration
+
+            address = device.address
+
+        else:
+            competition = ("_" + code[-2:]).lower()
+            registration = code
+            address = 0
+
+        elapsed_time = datetime.utcnow() - aircraft_beacon.timestamp
+        elapsed_seconds = int(elapsed_time.total_seconds())
+
+        lines.append(
+            '   <m a="{0:.7f},{1:.7f},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13}"/>'.format(
+                aircraft_beacon.location.latitude,
+                aircraft_beacon.location.longitude,
+                competition,
+                registration,
+                int(aircraft_beacon.altitude),
+                utc_to_local(aircraft_beacon.timestamp).strftime("%H:%M:%S"),
+                elapsed_seconds,
+                int(aircraft_beacon.track),
+                int(aircraft_beacon.ground_speed),
+                int(aircraft_beacon.climb_rate * 10) / 10,
+                aircraft_beacon.aircraft_type,
+                aircraft_beacon.receiver_name,
+                address,
+                code,
+            )
+        )
+
+    lines.append("</markers>")
+    xml = "\n".join(lines)
+
+    return xml

app/backend/ognrange.py

@@ -0,0 +1,54 @@
+import json
+from datetime import datetime, timedelta
+
+from sqlalchemy import func, case
+from sqlalchemy.sql.expression import label
+from app.model import Receiver, ReceiverCoverage
+
+from app import db
+
+
+def alchemyencoder(obj):
+    """JSON encoder function for SQLAlchemy special classes."""
+
+    import decimal
+    from datetime import datetime
+
+    if isinstance(obj, datetime):
+        return obj.strftime("%Y-%m-%d %H:%M")
+    elif isinstance(obj, decimal.Decimal):
+        return float(obj)
+
+
+def stations2_filtered_pl(start, end):
+    last_10_minutes = datetime.utcnow() - timedelta(minutes=10)
+
+    query = (
+        db.session.query(
+            Receiver.name.label("s"),
+            label("lt", func.round(func.ST_Y(Receiver.location_wkt) * 10000) / 10000),
+            label("lg", func.round(func.ST_X(Receiver.location_wkt) * 10000) / 10000),
+            case([(Receiver.lastseen > last_10_minutes, "U")], else_="D").label("u"),
+            Receiver.lastseen.label("ut"),
+            label("v", Receiver.version + "." + Receiver.platform),
+        )
+        .order_by(Receiver.lastseen)
+        .filter(db.or_(db.and_(start < Receiver.firstseen, end > Receiver.firstseen), db.and_(start < Receiver.lastseen, end > Receiver.lastseen)))
+    )
+
+    res = db.session.execute(query)
+    stations = json.dumps({"stations": [dict(r) for r in res]}, default=alchemyencoder)
+
+    return stations
+
+
+def max_tile_mgrs_pl(station, start, end, squares):
+    query = (
+        db.session.query(func.right(ReceiverCoverage.location_mgrs_short, 4), func.count(ReceiverCoverage.location_mgrs_short))
+        .filter(db.and_(Receiver.id == ReceiverCoverage.receiver_id, Receiver.name == station))
+        .filter(ReceiverCoverage.location_mgrs_short.like(squares + "%"))
+        .group_by(func.right(ReceiverCoverage.location_mgrs_short, 4))
+    )
+
+    res = {"t": squares, "p": ["{}/{}".format(r[0], r[1]) for r in query.all()]}
+    return json.dumps(res)

app/collect/celery.py

@@ -2,26 +2,26 @@ import datetime
 
 from celery.utils.log import get_task_logger
 
-from ogn_python.collect.takeoff_landings import update_entries as takeoff_update_entries
+from app.collect.takeoff_landings import update_entries as takeoff_update_entries
 
-from ogn_python.collect.logbook import update_entries as logbook_update_entries
-from ogn_python.collect.logbook import update_max_altitudes as logbook_update_max_altitudes
+from app.collect.logbook import update_entries as logbook_update_entries
+from app.collect.logbook import update_max_altitudes as logbook_update_max_altitudes
 
-from ogn_python.collect.database import import_ddb as device_infos_import_ddb
-from ogn_python.collect.database import update_country_code as receivers_update_country_code
+from app.collect.database import import_ddb as device_infos_import_ddb
+from app.collect.database import update_country_code as receivers_update_country_code
 
-from ogn_python.collect.stats import create_device_stats, update_device_stats_jumps, create_receiver_stats, create_relation_stats, update_qualities, update_receivers, update_devices
+from app.collect.stats import create_device_stats, update_device_stats_jumps, create_receiver_stats, create_relation_stats, update_qualities, update_receivers, update_devices
 
-from ogn_python.collect.ognrange import update_entries as receiver_coverage_update_entries
+from app.collect.ognrange import update_entries as receiver_coverage_update_entries
 
-from ogn_python import db
-from ogn_python import celery
+from app import db
+from app import celery
 
 
 logger = get_task_logger(__name__)
 
 
-@celery.task(name='update_takeoff_landings')
+@celery.task(name="update_takeoff_landings")
 def update_takeoff_landings(last_minutes):
     """Compute takeoffs and landings."""
 
@@ -31,7 +31,7 @@ def update_takeoff_landings(last_minutes):
     return result
 
 
-@celery.task(name='update_logbook_entries')
+@celery.task(name="update_logbook_entries")
 def update_logbook_entries(day_offset):
     """Add/update logbook entries."""
 
@@ -40,7 +40,7 @@ def update_logbook_entries(day_offset):
     return result
 
 
-@celery.task(name='update_logbook_max_altitude')
+@celery.task(name="update_logbook_max_altitude")
 def update_logbook_max_altitude(day_offset):
     """Add max altitudes in logbook when flight is complete (takeoff and landing)."""
 
@@ -49,7 +49,7 @@ def update_logbook_max_altitude(day_offset):
     return result
 
 
-@celery.task(name='import_ddb')
+@celery.task(name="import_ddb")
 def import_ddb():
     """Import registered devices from the DDB."""
 
@@ -57,7 +57,7 @@ def import_ddb():
     return result
 
 
-@celery.task(name='update_receivers_country_code')
+@celery.task(name="update_receivers_country_code")
 def update_receivers_country_code():
     """Update country code in receivers table if None."""
 
@@ -65,19 +65,16 @@ def update_receivers_country_code():
     return result
 
 
-@celery.task(name='purge_old_data')
+@celery.task(name="purge_old_data")
 def purge_old_data(max_hours):
     """Delete AircraftBeacons and ReceiverBeacons older than given 'age'."""
 
-    from ogn_python.model import AircraftBeacon, ReceiverBeacon
-    min_timestamp = datetime.datetime.utcnow() - datetime.timedelta(hours=max_hours)
-    aircraft_beacons_deleted = db.session.query(AircraftBeacon) \
-        .filter(AircraftBeacon.timestamp < min_timestamp) \
-        .delete()
+    from app.model import AircraftBeacon, ReceiverBeacon
 
-    receiver_beacons_deleted = db.session.query(ReceiverBeacon) \
-        .filter(ReceiverBeacon.timestamp < min_timestamp) \
-        .delete()
+    min_timestamp = datetime.datetime.utcnow() - datetime.timedelta(hours=max_hours)
+    aircraft_beacons_deleted = db.session.query(AircraftBeacon).filter(AircraftBeacon.timestamp < min_timestamp).delete()
+
+    receiver_beacons_deleted = db.session.query(ReceiverBeacon).filter(ReceiverBeacon.timestamp < min_timestamp).delete()
 
     db.session.commit()
 
@@ -85,7 +82,7 @@ def purge_old_data(max_hours):
     return result
 
 
-@celery.task(name='update_stats')
+@celery.task(name="update_stats")
 def update_stats(day_offset):
     """Create stats and update receivers/devices with stats."""
 
@@ -100,7 +97,7 @@ def update_stats(day_offset):
     update_devices(session=db.session)
 
 
-@celery.task(name='update_ognrange')
+@celery.task(name="update_ognrange")
 def update_ognrange(day_offset):
     """Create receiver coverage stats for Melissas ognrange."""
 

app/collect/database.py

@@ -3,10 +3,10 @@ from sqlalchemy.sql import null, and_, func, not_, case
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.dialects.postgresql import insert
 
-from ogn_python.model import Country, DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
-from ogn_python.utils import get_ddb, get_flarmnet
+from app.model import Country, DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
+from app.utils import get_ddb, get_flarmnet
 
-from ogn_python import app
+from app import app
 
 
 def upsert(session, model, rows, update_cols):
@@ -17,8 +17,7 @@ def upsert(session, model, rows, update_cols):
     stmt = insert(table).values(rows)
 
     on_conflict_stmt = stmt.on_conflict_do_update(
-        index_elements=table.primary_key.columns,
-        set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols},
+        index_elements=table.primary_key.columns, set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols}
     )
 
     # print(compile_query(on_conflict_stmt))
@@ -31,9 +30,7 @@ def update_device_infos(session, address_origin, path=None):
     else:
         device_infos = get_ddb(csv_file=path)
 
-    session.query(DeviceInfo) \
-        .filter(DeviceInfo.address_origin == address_origin) \
-        .delete(synchronize_session='fetch')
+    session.query(DeviceInfo).filter(DeviceInfo.address_origin == address_origin).delete(synchronize_session="fetch")
     session.commit()
 
     for device_info in device_infos:
@@ -65,10 +62,11 @@ def update_country_code(session, logger=None):
     if logger is None:
         logger = app.logger
 
-    update_receivers = session.query(Receiver) \
-        .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom))) \
-        .update({Receiver.country_id: Country.gid},
-                synchronize_session='fetch')
+    update_receivers = (
+        session.query(Receiver)
+        .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom)))
+        .update({Receiver.country_id: Country.gid}, synchronize_session="fetch")
+    )
 
     session.commit()
 

app/collect/logbook.py

@@ -0,0 +1,200 @@
+from sqlalchemy import and_, or_, insert, update, exists, between
+from sqlalchemy.sql import func, null
+from sqlalchemy.sql.expression import true, false
+
+from app.model import TakeoffLanding, Logbook, AircraftBeacon
+from app.utils import date_to_timestamps
+
+from app import app
+
+
+def update_entries(session, date, logger=None):
+    """Add/update logbook entries."""
+
+    if logger is None:
+        logger = app.logger
+
+    logger.info("Compute logbook.")
+
+    # limit time range to given date and set window partition and window order
+    (start, end) = date_to_timestamps(date)
+    pa = TakeoffLanding.device_id
+    wo = and_(TakeoffLanding.device_id, TakeoffLanding.airport_id, TakeoffLanding.timestamp)
+
+    # make a query with current, previous and next "takeoff_landing" event, so we can find complete flights
+    sq = (
+        session.query(
+            TakeoffLanding.device_id,
+            func.lag(TakeoffLanding.device_id).over(partition_by=pa, order_by=wo).label("device_id_prev"),
+            func.lead(TakeoffLanding.device_id).over(partition_by=pa, order_by=wo).label("device_id_next"),
+            TakeoffLanding.timestamp,
+            func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"),
+            func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"),
+            TakeoffLanding.track,
+            func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"),
+            func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"),
+            TakeoffLanding.is_takeoff,
+            func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"),
+            func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"),
+            TakeoffLanding.airport_id,
+            func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"),
+            func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next"),
+        )
+        .filter(between(TakeoffLanding.timestamp, start, end))
+        .subquery()
+    )
+
+    # find complete flights
+    complete_flight_query = session.query(
+        sq.c.timestamp.label("reftime"),
+        sq.c.device_id.label("device_id"),
+        sq.c.timestamp.label("takeoff_timestamp"),
+        sq.c.track.label("takeoff_track"),
+        sq.c.airport_id.label("takeoff_airport_id"),
+        sq.c.timestamp_next.label("landing_timestamp"),
+        sq.c.track_next.label("landing_track"),
+        sq.c.airport_id_next.label("landing_airport_id"),
+    ).filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false()))
+
+    # find landings without start
+    only_landings_query = (
+        session.query(
+            sq.c.timestamp.label("reftime"),
+            sq.c.device_id.label("device_id"),
+            null().label("takeoff_timestamp"),
+            null().label("takeoff_track"),
+            null().label("takeoff_airport_id"),
+            sq.c.timestamp.label("landing_timestamp"),
+            sq.c.track.label("landing_track"),
+            sq.c.airport_id.label("landing_airport_id"),
+        )
+        .filter(sq.c.is_takeoff == false())
+        .filter(or_(sq.c.is_takeoff_prev == false(), sq.c.is_takeoff_prev == null()))
+    )
+
+    # find starts without landing
+    only_starts_query = (
+        session.query(
+            sq.c.timestamp.label("reftime"),
+            sq.c.device_id.label("device_id"),
+            sq.c.timestamp.label("takeoff_timestamp"),
+            sq.c.track.label("takeoff_track"),
+            sq.c.airport_id.label("takeoff_airport_id"),
+            null().label("landing_timestamp"),
+            null().label("landing_track"),
+            null().label("landing_airport_id"),
+        )
+        .filter(sq.c.is_takeoff == true())
+        .filter(or_(sq.c.is_takeoff_next == true(), sq.c.is_takeoff_next == null()))
+    )
+
+    # unite all computated flights
+    union_query = complete_flight_query.union(only_landings_query, only_starts_query).subquery()
+
+    # if a logbook entry exist --> update it
+    upd = (
+        update(Logbook)
+        .where(
+            and_(
+                Logbook.device_id == union_query.c.device_id,
+                union_query.c.takeoff_airport_id != null(),
+                union_query.c.landing_airport_id != null(),
+                or_(
+                    and_(Logbook.takeoff_airport_id == union_query.c.takeoff_airport_id, Logbook.takeoff_timestamp == union_query.c.takeoff_timestamp, Logbook.landing_airport_id == null()),
+                    and_(Logbook.takeoff_airport_id == null(), Logbook.landing_airport_id == union_query.c.landing_airport_id, Logbook.landing_timestamp == union_query.c.landing_timestamp),
+                ),
+            )
+        )
+        .values(
+            {
+                "reftime": union_query.c.reftime,
+                "takeoff_timestamp": union_query.c.takeoff_timestamp,
+                "takeoff_track": union_query.c.takeoff_track,
+                "takeoff_airport_id": union_query.c.takeoff_airport_id,
+                "landing_timestamp": union_query.c.landing_timestamp,
+                "landing_track": union_query.c.landing_track,
+                "landing_airport_id": union_query.c.landing_airport_id,
+            }
+        )
+    )
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.debug("Updated logbook entries: {}".format(update_counter))
+
+    # if a logbook entry doesnt exist --> insert it
+    new_logbook_entries = session.query(union_query).filter(
+        ~exists().where(
+            and_(
+                Logbook.device_id == union_query.c.device_id,
+                or_(
+                    and_(Logbook.takeoff_airport_id == union_query.c.takeoff_airport_id, Logbook.takeoff_timestamp == union_query.c.takeoff_timestamp),
+                    and_(Logbook.takeoff_airport_id == null(), union_query.c.takeoff_airport_id == null()),
+                ),
+                or_(
+                    and_(Logbook.landing_airport_id == union_query.c.landing_airport_id, Logbook.landing_timestamp == union_query.c.landing_timestamp),
+                    and_(Logbook.landing_airport_id == null(), union_query.c.landing_airport_id == null()),
+                ),
+            )
+        )
+    )
+
+    ins = insert(Logbook).from_select(
+        (
+            Logbook.reftime,
+            Logbook.device_id,
+            Logbook.takeoff_timestamp,
+            Logbook.takeoff_track,
+            Logbook.takeoff_airport_id,
+            Logbook.landing_timestamp,
+            Logbook.landing_track,
+            Logbook.landing_airport_id,
+        ),
+        new_logbook_entries,
+    )
+
+    result = session.execute(ins)
+    insert_counter = result.rowcount
+    session.commit()
+
+    finish_message = "Logbook: {} inserted, {} updated".format(insert_counter, update_counter)
+    logger.debug(finish_message)
+    return finish_message
+
+
+def update_max_altitudes(session, date, logger=None):
+    """Add max altitudes in logbook when flight is complete (takeoff and landing)."""
+
+    if logger is None:
+        logger = app.logger
+
+    logger.info("Update logbook max altitude.")
+
+    if session is None:
+        session = app.session
+
+    (start, end) = date_to_timestamps(date)
+
+    logbook_entries = (
+        session.query(Logbook.id)
+        .filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null()))
+        .filter(between(Logbook.reftime, start, end))
+        .subquery()
+    )
+
+    max_altitudes = (
+        session.query(Logbook.id, func.max(AircraftBeacon.altitude).label("max_altitude"))
+        .filter(Logbook.id == logbook_entries.c.id)
+        .filter(and_(AircraftBeacon.device_id == Logbook.device_id, AircraftBeacon.timestamp >= Logbook.takeoff_timestamp, AircraftBeacon.timestamp <= Logbook.landing_timestamp))
+        .group_by(Logbook.id)
+        .subquery()
+    )
+
+    update_logbook = session.query(Logbook).filter(Logbook.id == max_altitudes.c.id).update({Logbook.max_altitude: max_altitudes.c.max_altitude}, synchronize_session="fetch")
+
+    session.commit()
+
+    finish_message = "Logbook (altitude): {} entries updated.".format(update_logbook)
+    logger.info(finish_message)
+    return finish_message

app/collect/ognrange.py

@@ -0,0 +1,89 @@
+from sqlalchemy import Date
+from sqlalchemy import and_, insert, update, exists, between
+from sqlalchemy.sql import func, null
+
+from app.model import AircraftBeacon, ReceiverCoverage
+from app.utils import date_to_timestamps
+
+from app import app
+
+
+def update_entries(session, date, logger=None):
+    """Create receiver coverage stats for Melissas ognrange."""
+
+    if logger is None:
+        logger = app.logger
+
+    logger.info("Compute receiver coverages.")
+
+    (start, end) = date_to_timestamps(date)
+
+    # Filter aircraft beacons
+    sq = (
+        session.query(AircraftBeacon.location_mgrs_short, AircraftBeacon.receiver_id, AircraftBeacon.signal_quality, AircraftBeacon.altitude, AircraftBeacon.device_id)
+        .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.location_mgrs_short != null(), AircraftBeacon.receiver_id != null(), AircraftBeacon.device_id != null()))
+        .subquery()
+    )
+
+    # ... and group them by reduced MGRS, receiver and date
+    query = (
+        session.query(
+            sq.c.location_mgrs_short,
+            sq.c.receiver_id,
+            func.cast(date, Date).label("date"),
+            func.max(sq.c.signal_quality).label("max_signal_quality"),
+            func.min(sq.c.altitude).label("min_altitude"),
+            func.max(sq.c.altitude).label("max_altitude"),
+            func.count(sq.c.altitude).label("aircraft_beacon_count"),
+            func.count(func.distinct(sq.c.device_id)).label("device_count"),
+        )
+        .group_by(sq.c.location_mgrs_short, sq.c.receiver_id)
+        .subquery()
+    )
+
+    # if a receiver coverage entry exist --> update it
+    upd = (
+        update(ReceiverCoverage)
+        .where(and_(ReceiverCoverage.location_mgrs_short == query.c.location_mgrs_short, ReceiverCoverage.receiver_id == query.c.receiver_id, ReceiverCoverage.date == date))
+        .values(
+            {
+                "max_signal_quality": query.c.max_signal_quality,
+                "min_altitude": query.c.min_altitude,
+                "max_altitude": query.c.max_altitude,
+                "aircraft_beacon_count": query.c.aircraft_beacon_count,
+                "device_count": query.c.device_count,
+            }
+        )
+    )
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.debug("Updated receiver coverage entries: {}".format(update_counter))
+
+    # if a receiver coverage entry doesnt exist --> insert it
+    new_coverage_entries = session.query(query).filter(
+        ~exists().where(and_(ReceiverCoverage.location_mgrs_short == query.c.location_mgrs_short, ReceiverCoverage.receiver_id == query.c.receiver_id, ReceiverCoverage.date == date))
+    )
+
+    ins = insert(ReceiverCoverage).from_select(
+        (
+            ReceiverCoverage.location_mgrs_short,
+            ReceiverCoverage.receiver_id,
+            ReceiverCoverage.date,
+            ReceiverCoverage.max_signal_quality,
+            ReceiverCoverage.min_altitude,
+            ReceiverCoverage.max_altitude,
+            ReceiverCoverage.aircraft_beacon_count,
+            ReceiverCoverage.device_count,
+        ),
+        new_coverage_entries,
+    )
+
+    result = session.execute(ins)
+    insert_counter = result.rowcount
+    session.commit()
+
+    finish_message = "ReceiverCoverage: {} inserted, {} updated".format(insert_counter, update_counter)
+    logger.debug(finish_message)
+    return finish_message

app/collect/stats.py

@@ -0,0 +1,451 @@
+from sqlalchemy import insert, distinct, between, literal
+from sqlalchemy.sql import null, and_, func, or_, update
+from sqlalchemy.sql.expression import case
+
+from app.model import AircraftBeacon, DeviceStats, Country, CountryStats, ReceiverStats, ReceiverBeacon, RelationStats, Receiver, Device
+
+from app.utils import date_to_timestamps
+
+from app import app
+
+# 40dB@10km is enough for 640km
+MAX_PLAUSIBLE_QUALITY = 40
+
+
+def create_device_stats(session, date, logger=None):
+    """Add/update device stats."""
+
+    if logger is None:
+        logger = app.logger
+
+    (start, end) = date_to_timestamps(date)
+
+    # First kill the stats for the selected date
+    deleted_counter = session.query(DeviceStats).filter(DeviceStats.date == date).delete()
+
+    # Since "distinct count" does not work in window functions we need a work-around for receiver counting
+    sq = (
+        session.query(AircraftBeacon, func.dense_rank().over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id).label("dr"))
+        .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.device_id != null()))
+        .filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null()))
+        .subquery()
+    )
+
+    # Calculate stats, firstseen, lastseen and last values != NULL
+    device_stats = session.query(
+        distinct(sq.c.device_id).label("device_id"),
+        literal(date).label("date"),
+        func.max(sq.c.dr).over(partition_by=sq.c.device_id).label("receiver_count"),
+        func.max(sq.c.altitude).over(partition_by=sq.c.device_id).label("max_altitude"),
+        func.count(sq.c.device_id).over(partition_by=sq.c.device_id).label("aircraft_beacon_count"),
+        func.first_value(sq.c.name).over(partition_by=sq.c.device_id, order_by=case([(sq.c.name == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("name"),
+        func.first_value(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("firstseen"),
+        func.first_value(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("lastseen"),
+        func.first_value(sq.c.aircraft_type).over(partition_by=sq.c.device_id, order_by=case([(sq.c.aircraft_type == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("aircraft_type"),
+        func.first_value(sq.c.stealth).over(partition_by=sq.c.device_id, order_by=case([(sq.c.stealth == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("stealth"),
+        func.first_value(sq.c.software_version)
+        .over(partition_by=sq.c.device_id, order_by=case([(sq.c.software_version == null(), None)], else_=sq.c.timestamp).desc().nullslast())
+        .label("software_version"),
+        func.first_value(sq.c.hardware_version)
+        .over(partition_by=sq.c.device_id, order_by=case([(sq.c.hardware_version == null(), None)], else_=sq.c.timestamp).desc().nullslast())
+        .label("hardware_version"),
+        func.first_value(sq.c.real_address).over(partition_by=sq.c.device_id, order_by=case([(sq.c.real_address == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("real_address"),
+    ).subquery()
+
+    # And insert them
+    ins = insert(DeviceStats).from_select(
+        [
+            DeviceStats.device_id,
+            DeviceStats.date,
+            DeviceStats.receiver_count,
+            DeviceStats.max_altitude,
+            DeviceStats.aircraft_beacon_count,
+            DeviceStats.name,
+            DeviceStats.firstseen,
+            DeviceStats.lastseen,
+            DeviceStats.aircraft_type,
+            DeviceStats.stealth,
+            DeviceStats.software_version,
+            DeviceStats.hardware_version,
+            DeviceStats.real_address,
+        ],
+        device_stats,
+    )
+    res = session.execute(ins)
+    insert_counter = res.rowcount
+    session.commit()
+    logger.debug("DeviceStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
+
+    return "DeviceStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
+
+
+def create_receiver_stats(session, date, logger=None):
+    """Add/update receiver stats."""
+
+    if logger is None:
+        logger = app.logger
+
+    (start, end) = date_to_timestamps(date)
+
+    # First kill the stats for the selected date
+    deleted_counter = session.query(ReceiverStats).filter(ReceiverStats.date == date).delete()
+
+    # Select one day
+    sq = session.query(ReceiverBeacon).filter(between(ReceiverBeacon.timestamp, start, end)).subquery()
+
+    # Calculate stats, firstseen, lastseen and last values != NULL
+    receiver_stats = session.query(
+        distinct(sq.c.receiver_id).label("receiver_id"),
+        literal(date).label("date"),
+        func.first_value(sq.c.timestamp).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("firstseen"),
+        func.first_value(sq.c.timestamp).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("lastseen"),
+        func.first_value(sq.c.location).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.location == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("location_wkt"),
+        func.first_value(sq.c.altitude).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.altitude == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("altitude"),
+        func.first_value(sq.c.version).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.version == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("version"),
+        func.first_value(sq.c.platform).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.platform == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("platform"),
+    ).subquery()
+
+    # And insert them
+    ins = insert(ReceiverStats).from_select(
+        [
+            ReceiverStats.receiver_id,
+            ReceiverStats.date,
+            ReceiverStats.firstseen,
+            ReceiverStats.lastseen,
+            ReceiverStats.location_wkt,
+            ReceiverStats.altitude,
+            ReceiverStats.version,
+            ReceiverStats.platform,
+        ],
+        receiver_stats,
+    )
+    res = session.execute(ins)
+    insert_counter = res.rowcount
+    session.commit()
+    logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
+
+    # Update aircraft_beacon_count, aircraft_count and max_distance
+    aircraft_beacon_stats = (
+        session.query(
+            AircraftBeacon.receiver_id,
+            func.count(AircraftBeacon.timestamp).label("aircraft_beacon_count"),
+            func.count(func.distinct(AircraftBeacon.device_id)).label("aircraft_count"),
+            func.max(AircraftBeacon.distance).label("max_distance"),
+        )
+        .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.error_count == 0, AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY, AircraftBeacon.relay == null()))
+        .group_by(AircraftBeacon.receiver_id)
+        .subquery()
+    )
+
+    upd = (
+        update(ReceiverStats)
+        .where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id))
+        .values(
+            {"aircraft_beacon_count": aircraft_beacon_stats.c.aircraft_beacon_count, "aircraft_count": aircraft_beacon_stats.c.aircraft_count, "max_distance": aircraft_beacon_stats.c.max_distance}
+        )
+    )
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.warn("Updated {} ReceiverStats".format(update_counter))
+
+    return "ReceiverStats for {}: {} deleted, {} inserted, {} updated".format(date, deleted_counter, insert_counter, update_counter)
+
+
+def create_country_stats(session, date, logger=None):
+    if logger is None:
+        logger = app.logger
+
+    (start, end) = date_to_timestamps(date)
+
+    # First kill the stats for the selected date
+    deleted_counter = session.query(CountryStats).filter(CountryStats.date == date).delete()
+
+    country_stats = (
+        session.query(literal(date), Country.gid, func.count(AircraftBeacon.timestamp).label("aircraft_beacon_count"), func.count(func.distinct(AircraftBeacon.receiver_id)).label("device_count"))
+        .filter(between(AircraftBeacon.timestamp, start, end))
+        .filter(func.st_contains(Country.geom, AircraftBeacon.location))
+        .group_by(Country.gid)
+        .subquery()
+    )
+
+    # And insert them
+    ins = insert(CountryStats).from_select([CountryStats.date, CountryStats.country_id, CountryStats.aircraft_beacon_count, CountryStats.device_count], country_stats)
+    res = session.execute(ins)
+    insert_counter = res.rowcount
+    session.commit()
+
+
+def update_device_stats_jumps(session, date, logger=None):
+    """Update device stats jumps."""
+
+    if logger is None:
+        logger = app.logger
+
+    (start, end) = date_to_timestamps(date)
+
+    # speed limits in m/s (values above indicates a unplausible position / jump)
+    max_horizontal_speed = 1000
+    max_vertical_speed = 100
+    max_jumps = 10  # threshold for an 'ambiguous' device
+
+    # find consecutive positions for a device
+    sq = (
+        session.query(
+            AircraftBeacon.device_id,
+            AircraftBeacon.timestamp,
+            func.lead(AircraftBeacon.timestamp).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("timestamp_next"),
+            AircraftBeacon.location_wkt,
+            func.lead(AircraftBeacon.location_wkt).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("location_next"),
+            AircraftBeacon.altitude,
+            func.lead(AircraftBeacon.altitude).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("altitude_next"),
+        )
+        .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.error_count == 0))
+        .subquery()
+    )
+
+    # calc vertial and horizontal speed between points
+    sq2 = (
+        session.query(
+            sq.c.device_id,
+            (func.st_distancesphere(sq.c.location_next, sq.c.location) / (func.extract("epoch", sq.c.timestamp_next) - func.extract("epoch", sq.c.timestamp))).label("horizontal_speed"),
+            ((sq.c.altitude_next - sq.c.altitude) / (func.extract("epoch", sq.c.timestamp_next) - func.extract("epoch", sq.c.timestamp))).label("vertical_speed"),
+        )
+        .filter(and_(sq.c.timestamp != null(), sq.c.timestamp_next != null(), sq.c.timestamp < sq.c.timestamp_next))
+        .subquery()
+    )
+
+    # ... and find and count 'jumps'
+    sq3 = (
+        session.query(sq2.c.device_id, func.sum(case([(or_(func.abs(sq2.c.horizontal_speed) > max_horizontal_speed, func.abs(sq2.c.vertical_speed) > max_vertical_speed), 1)], else_=0)).label("jumps"))
+        .group_by(sq2.c.device_id)
+        .subquery()
+    )
+
+    upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == sq3.c.device_id)).values({"ambiguous": sq3.c.jumps > max_jumps, "jumps": sq3.c.jumps})
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.warn("Updated {} DeviceStats jumps".format(update_counter))
+
+    return "DeviceStats jumps for {}: {} updated".format(date, update_counter)
+
+
+def create_relation_stats(session, date, logger=None):
+    """Add/update relation stats."""
+
+    if logger is None:
+        logger = app.logger
+
+    (start, end) = date_to_timestamps(date)
+
+    # First kill the stats for the selected date
+    deleted_counter = session.query(RelationStats).filter(RelationStats.date == date).delete()
+
+    # Calculate stats for selected day
+    relation_stats = (
+        session.query(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id, func.max(AircraftBeacon.quality), func.count(AircraftBeacon.timestamp))
+        .filter(
+            and_(
+                between(AircraftBeacon.timestamp, start, end),
+                AircraftBeacon.distance > 1000,
+                AircraftBeacon.error_count == 0,
+                AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY,
+                AircraftBeacon.ground_speed > 10,
+            )
+        )
+        .group_by(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id)
+        .subquery()
+    )
+
+    # And insert them
+    ins = insert(RelationStats).from_select([RelationStats.date, RelationStats.device_id, RelationStats.receiver_id, RelationStats.quality, RelationStats.beacon_count], relation_stats)
+    res = session.execute(ins)
+    insert_counter = res.rowcount
+    session.commit()
+    logger.warn("RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter))
+
+    return "RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)
+
+
+def update_qualities(session, date, logger=None):
+    """Calculate relative qualities of receivers and devices."""
+
+    if logger is None:
+        logger = app.logger
+
+    # Calculate avg quality of devices
+    dev_sq = session.query(RelationStats.device_id, func.avg(RelationStats.quality).label("quality")).filter(RelationStats.date == date).group_by(RelationStats.device_id).subquery()
+
+    dev_upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)).values({"quality": dev_sq.c.quality})
+
+    dev_result = session.execute(dev_upd)
+    dev_update_counter = dev_result.rowcount
+    session.commit()
+    logger.warn("Updated {} DeviceStats: quality".format(dev_update_counter))
+
+    # Calculate avg quality of receivers
+    rec_sq = session.query(RelationStats.receiver_id, func.avg(RelationStats.quality).label("quality")).filter(RelationStats.date == date).group_by(RelationStats.receiver_id).subquery()
+
+    rec_upd = update(ReceiverStats).where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)).values({"quality": rec_sq.c.quality})
+
+    rec_result = session.execute(rec_upd)
+    rec_update_counter = rec_result.rowcount
+    session.commit()
+    logger.warn("Updated {} ReceiverStats: quality".format(rec_update_counter))
+
+    # Calculate quality_offset of devices
+    dev_sq = (
+        session.query(
+            RelationStats.device_id, (func.sum(RelationStats.beacon_count * (RelationStats.quality - ReceiverStats.quality)) / (func.sum(RelationStats.beacon_count))).label("quality_offset")
+        )
+        .filter(RelationStats.date == date)
+        .filter(and_(RelationStats.receiver_id == ReceiverStats.receiver_id, RelationStats.date == ReceiverStats.date))
+        .group_by(RelationStats.device_id)
+        .subquery()
+    )
+
+    dev_upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)).values({"quality_offset": dev_sq.c.quality_offset})
+
+    dev_result = session.execute(dev_upd)
+    dev_update_counter = dev_result.rowcount
+    session.commit()
+    logger.warn("Updated {} DeviceStats: quality_offset".format(dev_update_counter))
+
+    # Calculate quality_offset of receivers
+    rec_sq = (
+        session.query(
+            RelationStats.receiver_id, (func.sum(RelationStats.beacon_count * (RelationStats.quality - DeviceStats.quality)) / (func.sum(RelationStats.beacon_count))).label("quality_offset")
+        )
+        .filter(RelationStats.date == date)
+        .filter(and_(RelationStats.device_id == DeviceStats.device_id, RelationStats.date == DeviceStats.date))
+        .group_by(RelationStats.receiver_id)
+        .subquery()
+    )
+
+    rec_upd = update(ReceiverStats).where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)).values({"quality_offset": rec_sq.c.quality_offset})
+
+    rec_result = session.execute(rec_upd)
+    rec_update_counter = rec_result.rowcount
+    session.commit()
+    logger.warn("Updated {} ReceiverStats: quality_offset".format(rec_update_counter))
+
+    return "Updated {} DeviceStats and {} ReceiverStats".format(dev_update_counter, rec_update_counter)
+
+
+def update_receivers(session, logger=None):
+    """Update receivers with stats."""
+
+    if logger is None:
+        logger = app.logger
+
+    receiver_stats = (
+        session.query(
+            distinct(ReceiverStats.receiver_id).label("receiver_id"),
+            func.first_value(ReceiverStats.firstseen)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.firstseen == null(), None)], else_=ReceiverStats.date).asc().nullslast())
+            .label("firstseen"),
+            func.first_value(ReceiverStats.lastseen)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.lastseen == null(), None)], else_=ReceiverStats.date).desc().nullslast())
+            .label("lastseen"),
+            func.first_value(ReceiverStats.location_wkt)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.location_wkt == null(), None)], else_=ReceiverStats.date).desc().nullslast())
+            .label("location_wkt"),
+            func.first_value(ReceiverStats.altitude)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.altitude == null(), None)], else_=ReceiverStats.date).desc().nullslast())
+            .label("altitude"),
+            func.first_value(ReceiverStats.version)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.version == null(), None)], else_=ReceiverStats.date).desc().nullslast())
+            .label("version"),
+            func.first_value(ReceiverStats.platform)
+            .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.platform == null(), None)], else_=ReceiverStats.date).desc().nullslast())
+            .label("platform"),
+        )
+        .order_by(ReceiverStats.receiver_id)
+        .subquery()
+    )
+
+    upd = (
+        update(Receiver)
+        .where(and_(Receiver.id == receiver_stats.c.receiver_id))
+        .values(
+            {
+                "firstseen": receiver_stats.c.firstseen,
+                "lastseen": receiver_stats.c.lastseen,
+                "location": receiver_stats.c.location_wkt,
+                "altitude": receiver_stats.c.altitude,
+                "version": receiver_stats.c.version,
+                "platform": receiver_stats.c.platform,
+            }
+        )
+    )
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.warn("Updated {} Receivers".format(update_counter))
+
+    return "Updated {} Receivers".format(update_counter)
+
+
+def update_devices(session, logger=None):
+    """Update devices with stats."""
+
+    if logger is None:
+        logger = app.logger
+
+    device_stats = (
+        session.query(
+            distinct(DeviceStats.device_id).label("device_id"),
+            func.first_value(DeviceStats.name).over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.name == null(), None)], else_=DeviceStats.date).desc().nullslast()).label("name"),
+            func.first_value(DeviceStats.firstseen)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.firstseen == null(), None)], else_=DeviceStats.date).asc().nullslast())
+            .label("firstseen"),
+            func.max(DeviceStats.lastseen)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.lastseen == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("lastseen"),
+            func.first_value(DeviceStats.aircraft_type)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.aircraft_type == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("aircraft_type"),
+            func.first_value(DeviceStats.stealth)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.stealth == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("stealth"),
+            func.first_value(DeviceStats.software_version)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.software_version == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("software_version"),
+            func.first_value(DeviceStats.hardware_version)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.hardware_version == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("hardware_version"),
+            func.first_value(DeviceStats.real_address)
+            .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.real_address == null(), None)], else_=DeviceStats.date).desc().nullslast())
+            .label("real_address"),
+        )
+        .order_by(DeviceStats.device_id)
+        .subquery()
+    )
+
+    upd = (
+        update(Device)
+        .where(and_(Device.id == device_stats.c.device_id))
+        .values(
+            {
+                "name": device_stats.c.name,
+                "firstseen": device_stats.c.firstseen,
+                "lastseen": device_stats.c.lastseen,
+                "aircraft_type": device_stats.c.aircraft_type,
+                "stealth": device_stats.c.stealth,
+                "software_version": device_stats.c.software_version,
+                "hardware_version": device_stats.c.hardware_version,
+                "real_address": device_stats.c.real_address,
+            }
+        )
+    )
+
+    result = session.execute(upd)
+    update_counter = result.rowcount
+    session.commit()
+    logger.warn("Updated {} Devices".format(update_counter))
+
+    return "Updated {} Devices".format(update_counter)

app/collect/takeoff_landings.py

@@ -0,0 +1,144 @@
+from datetime import timedelta
+
+from sqlalchemy import and_, or_, insert, between, exists
+from sqlalchemy.sql import func, null
+from sqlalchemy.sql.expression import case
+
+from app.model import AircraftBeacon, TakeoffLanding, Airport
+
+from app import app
+
+
+def update_entries(session, start, end, logger=None):
+    """Compute takeoffs and landings."""
+
+    if logger is None:
+        logger = app.logger
+
+    logger.info("Compute takeoffs and landings.")
+
+    # considered time interval should not exceed a complete day
+    if end - start > timedelta(days=1):
+        abort_message = "TakeoffLanding: timeinterval start='{}' and end='{}' is too big.".format(start, end)
+        logger.warn(abort_message)
+        return abort_message
+
+    # check if we have any airport
+    airports_query = session.query(Airport).limit(1)
+    if not airports_query.all():
+        abort_message = "TakeoffLanding: Cannot calculate takeoff and landings without any airport! Please import airports first."
+        logger.warn(abort_message)
+        return abort_message
+
+    # takeoff / landing detection is based on 3 consecutive points all below a certain altitude AGL
+    takeoff_speed = 55  # takeoff detection: 1st point below, 2nd and 3rd above this limit
+    landing_speed = 40  # landing detection: 1st point above, 2nd and 3rd below this limit
+    min_takeoff_climb_rate = -5  # takeoff detection: glider should not sink too much
+    max_landing_climb_rate = 5  # landing detection: glider should not climb too much
+    duration = 100  # the points must not exceed this duration
+    radius = 5000  # the points must not exceed this radius around the 2nd point
+    max_agl = 200  # takeoff / landing must not exceed this altitude AGL
+
+    # get beacons for selected time range, one per device_id and timestamp
+    sq = (
+        session.query(AircraftBeacon)
+        .distinct(AircraftBeacon.device_id, AircraftBeacon.timestamp)
+        .order_by(AircraftBeacon.device_id, AircraftBeacon.timestamp, AircraftBeacon.error_count)
+        .filter(AircraftBeacon.agl < max_agl)
+        .filter(between(AircraftBeacon.timestamp, start, end))
+        .subquery()
+    )
+
+    # make a query with current, previous and next position
+    sq2 = session.query(
+        sq.c.device_id,
+        func.lag(sq.c.device_id).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("device_id_prev"),
+        func.lead(sq.c.device_id).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("device_id_next"),
+        sq.c.timestamp,
+        func.lag(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("timestamp_prev"),
+        func.lead(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("timestamp_next"),
+        sq.c.location,
+        func.lag(sq.c.location).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("location_wkt_prev"),
+        func.lead(sq.c.location).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("location_wkt_next"),
+        sq.c.track,
+        func.lag(sq.c.track).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("track_prev"),
+        func.lead(sq.c.track).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("track_next"),
+        sq.c.ground_speed,
+        func.lag(sq.c.ground_speed).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("ground_speed_prev"),
+        func.lead(sq.c.ground_speed).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("ground_speed_next"),
+        sq.c.altitude,
+        func.lag(sq.c.altitude).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("altitude_prev"),
+        func.lead(sq.c.altitude).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("altitude_next"),
+        sq.c.climb_rate,
+        func.lag(sq.c.climb_rate).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("climb_rate_prev"),
+        func.lead(sq.c.climb_rate).over(partition_by=sq.c.device_id, order_by=sq.c.timestamp).label("climb_rate_next"),
+    ).subquery()
+
+    # consider only positions with predecessor and successor and limit distance and duration between points
+    sq3 = (
+        session.query(sq2)
+        .filter(and_(sq2.c.device_id_prev != null(), sq2.c.device_id_next != null()))
+        .filter(and_(func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < radius, func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < radius))
+        .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration))
+        .subquery()
+    )
+
+    # find possible takeoffs and landings
+    sq4 = (
+        session.query(
+            sq3.c.timestamp,
+            case(
+                [
+                    (sq3.c.ground_speed > takeoff_speed, sq3.c.location_wkt_prev),  # on takeoff we take the location from the previous fix because it is nearer to the airport
+                    (sq3.c.ground_speed <= takeoff_speed, sq3.c.location),
+                ]
+            ).label("location"),
+            case([(sq3.c.ground_speed > landing_speed, sq3.c.track), (sq3.c.ground_speed <= landing_speed, sq3.c.track_prev)]).label(
+                "track"
+            ),  # on landing we take the track from the previous fix because gliders tend to leave the runway quickly
+            sq3.c.ground_speed,
+            sq3.c.altitude,
+            case([(sq3.c.ground_speed > takeoff_speed, True), (sq3.c.ground_speed < landing_speed, False)]).label("is_takeoff"),
+            sq3.c.device_id,
+        )
+        .filter(
+            or_(
+                and_(sq3.c.ground_speed_prev < takeoff_speed, sq3.c.ground_speed > takeoff_speed, sq3.c.ground_speed_next > takeoff_speed, sq3.c.climb_rate > min_takeoff_climb_rate),  # takeoff
+                and_(sq3.c.ground_speed_prev > landing_speed, sq3.c.ground_speed < landing_speed, sq3.c.ground_speed_next < landing_speed, sq3.c.climb_rate < max_landing_climb_rate),  # landing
+            )
+        )
+        .subquery()
+    )
+
+    # consider them if the are near airports ...
+    sq5 = (
+        session.query(
+            sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, sq4.c.device_id, Airport.id.label("airport_id"), func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance")
+        )
+        .filter(and_(func.ST_Within(sq4.c.location, Airport.border), between(Airport.style, 2, 5)))
+        .subquery()
+    )
+
+    # ... and take the nearest airport
+    sq6 = (
+        session.query(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id, sq5.c.airport_id)
+        .distinct(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id)
+        .order_by(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id, sq5.c.airport_distance)
+        .subquery()
+    )
+
+    # consider them only if they are not already existing in db
+    takeoff_landing_query = session.query(sq6).filter(
+        ~exists().where(and_(TakeoffLanding.timestamp == sq6.c.timestamp, TakeoffLanding.device_id == sq6.c.device_id, TakeoffLanding.airport_id == sq6.c.airport_id))
+    )
+
+    # ... and save them
+    ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp, TakeoffLanding.track, TakeoffLanding.is_takeoff, TakeoffLanding.device_id, TakeoffLanding.airport_id), takeoff_landing_query)
+
+    result = session.execute(ins)
+    session.commit()
+    insert_counter = result.rowcount
+
+    finish_message = "TakeoffLandings: {} inserted".format(insert_counter)
+    logger.info(finish_message)
+    return finish_message

app/commands/__init__.py

@@ -1,4 +1,4 @@
-from ogn_python import app
+from app import app
 
 from .database import user_cli as database_cli
 from .export import user_cli as export_cli

app/commands/database.py

@@ -4,14 +4,14 @@ import click
 from datetime import datetime, timedelta
 from sqlalchemy.sql import func
 
-from ogn_python.collect.database import update_device_infos, update_country_code
-from ogn_python.model import *
-from ogn_python.utils import get_airports, get_days
+from app.collect.database import update_device_infos, update_country_code
+from app.model import *
+from app.utils import get_airports, get_days
 
-from ogn_python import app
-from ogn_python import db
+from app import app
+from app import db
 
-user_cli = AppGroup('database')
+user_cli = AppGroup("database")
 user_cli.help = "Database creation and handling."
 
 
@@ -22,7 +22,7 @@ def get_database_days(start, end):
     """Returns the first and the last day in aircraft_beacons table."""
 
     if start is None and end is None:
-        days_from_db = db.session.query(func.min(AircraftBeacon.timestamp).label('first_day'), func.max(AircraftBeacon.timestamp).label('last_day')).one()
+        days_from_db = db.session.query(func.min(AircraftBeacon.timestamp).label("first_day"), func.max(AircraftBeacon.timestamp).label("last_day")).one()
         start = days_from_db[0].date()
         end = days_from_db[1].date()
     else:
@@ -34,40 +34,40 @@ def get_database_days(start, end):
     return days
 
 
-@user_cli.command('info')
+@user_cli.command("info")
 def info():
     print(app.config)
-    print(app.config['SQLALCHEMY_DATABASE_URI'])
+    print(app.config["SQLALCHEMY_DATABASE_URI"])
 
 
-@user_cli.command('init')
+@user_cli.command("init")
 def init():
     """Initialize the database."""
 
     from alembic.config import Config
     from alembic import command
 
-    db.session.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
-    db.session.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;')
+    db.session.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
+    db.session.execute("CREATE EXTENSION IF NOT EXISTS btree_gist;")
     db.session.commit()
     db.create_all()
 
-    #alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
-    #command.stamp(alembic_cfg, "head")
+    # alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
+    # command.stamp(alembic_cfg, "head")
     print("Done.")
 
 
-@user_cli.command('init_timescaledb')
+@user_cli.command("init_timescaledb")
 def init_timescaledb():
     """Initialize TimescaleDB features."""
 
-    db.session.execute('CREATE EXTENSION IF NOT EXISTS timescaledb;')
+    db.session.execute("CREATE EXTENSION IF NOT EXISTS timescaledb;")
     db.session.execute("SELECT create_hypertable('aircraft_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);")
     db.session.execute("SELECT create_hypertable('receiver_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);")
     db.session.commit()
 
 
-@user_cli.command('upgrade')
+@user_cli.command("upgrade")
 def upgrade():
     """Upgrade database to the latest version."""
 
@@ -75,21 +75,21 @@ def upgrade():
     from alembic import command
 
     alembic_cfg = Config(ALEMBIC_CONFIG_FILE)
-    command.upgrade(alembic_cfg, 'head')
+    command.upgrade(alembic_cfg, "head")
 
 
-@user_cli.command('drop')
-@click.option('--sure', default='n')
+@user_cli.command("drop")
+@click.option("--sure", default="n")
 def drop(sure):
     """Drop all tables."""
-    if sure == 'y':
+    if sure == "y":
         db.drop_all()
-        print('Dropped all tables.')
+        print("Dropped all tables.")
     else:
         print("Add argument '--sure y' to drop all tables.")
 
 
-@user_cli.command('import_ddb')
+@user_cli.command("import_ddb")
 def import_ddb():
     """Import registered devices from the DDB."""
 
@@ -98,33 +98,29 @@ def import_ddb():
     print("Imported %i devices." % counter)
 
 
-@user_cli.command('import_file')
-@click.argument('path')
-def import_file(path='tests/custom_ddb.txt'):
+@user_cli.command("import_file")
+@click.argument("path")
+def import_file(path="tests/custom_ddb.txt"):
     """Import registered devices from a local file."""
 
     print("Import registered devices from '{}'...".format(path))
-    counter = update_device_infos(db.session,
-                                  DeviceInfoOrigin.user_defined,
-                                  path=path)
+    counter = update_device_infos(db.session, DeviceInfoOrigin.user_defined, path=path)
     print("Imported %i devices." % counter)
 
 
-@user_cli.command('import_flarmnet')
-@click.argument('path')
+@user_cli.command("import_flarmnet")
+@click.argument("path")
 def import_flarmnet(path=None):
     """Import registered devices from a local file."""
 
     print("Import registered devices from '{}'...".format("internet" if path is None else path))
-    counter = update_device_infos(db.session,
-                                  DeviceInfoOrigin.flarmnet,
-                                  path=path)
+    counter = update_device_infos(db.session, DeviceInfoOrigin.flarmnet, path=path)
     print("Imported %i devices." % counter)
 
 
-@user_cli.command('import_airports')
-@click.argument('path')
-def import_airports(path='tests/SeeYou.cup'):
+@user_cli.command("import_airports")
+@click.argument("path")
+def import_airports(path="tests/SeeYou.cup"):
     """Import airports from a ".cup" file"""
 
     print("Import airports from '{}'...".format(path))
@@ -136,7 +132,7 @@ def import_airports(path='tests/SeeYou.cup'):
     print("Imported {} airports.".format(len(airports)))
 
 
-@user_cli.command('update_country_codes')
+@user_cli.command("update_country_codes")
 def update_country_codes():
     """Update country codes of all receivers."""
 

app/commands/export.py

@@ -6,14 +6,14 @@ import re
 import csv
 
 from aerofiles.igc import Writer
-from ogn_python.model import *
-from ogn_python import db
+from app.model import *
+from app import db
 
-user_cli = AppGroup('export')
+user_cli = AppGroup("export")
 user_cli.help = "Export data in several file formats."
 
 
-@user_cli.command('cup')
+@user_cli.command("cup")
 def cup():
     """Export receiver waypoints as '.cup'."""
 
@@ -49,65 +49,61 @@ def cup():
         """
     results = db.session.execute(sql)
 
-    with open('receivers.cup', 'w') as outfile:
+    with open("receivers.cup", "w") as outfile:
         outcsv = csv.writer(outfile)
         outcsv.writerow(results.keys())
         outcsv.writerows(results.fetchall())
 
-@user_cli.command('igc')
-@click.argument('address')
-@click.argument('date')
+
+@user_cli.command("igc")
+@click.argument("address")
+@click.argument("date")
 def igc(address, date):
     """Export igc file for <address> at <date>."""
-    if not re.match('.{6}', address):
+    if not re.match(".{6}", address):
         print("Address {} not valid.".format(address))
         return
 
-    if not re.match('\d{4}-\d{2}-\d{2}', date):
+    if not re.match("\d{4}-\d{2}-\d{2}", date):
         print("Date {} not valid.".format(date))
         return
 
-    device_id = db.session.query(Device.id) \
-        .filter(Device.address == address) \
-        .first()
+    device_id = db.session.query(Device.id).filter(Device.address == address).first()
 
-    if (device_id is None):
+    if device_id is None:
         print("Device with address '{}' not found.".format(address))
         return
 
-    with open('sample.igc', 'wb') as fp:
+    with open("sample.igc", "wb") as fp:
         writer = Writer(fp)
 
-        writer.write_headers({
-            'manufacturer_code': 'OGN',
-            'logger_id': 'OGN',
-            'date': datetime.date(1987, 2, 24),
-            'fix_accuracy': 50,
-            'pilot': 'Konstantin Gruendger',
-            'copilot': '',
-            'glider_type': 'Duo Discus',
-            'glider_id': 'D-KKHH',
-            'firmware_version': '2.2',
-            'hardware_version': '2',
-            'logger_type': 'LXNAVIGATION,LX8000F',
-            'gps_receiver': 'uBLOX LEA-4S-2,16,max9000m',
-            'pressure_sensor': 'INTERSEMA,MS5534A,max10000m',
-            'competition_id': '2H',
-            'competition_class': 'Doubleseater',
-        })
+        writer.write_headers(
+            {
+                "manufacturer_code": "OGN",
+                "logger_id": "OGN",
+                "date": datetime.date(1987, 2, 24),
+                "fix_accuracy": 50,
+                "pilot": "Konstantin Gruendger",
+                "copilot": "",
+                "glider_type": "Duo Discus",
+                "glider_id": "D-KKHH",
+                "firmware_version": "2.2",
+                "hardware_version": "2",
+                "logger_type": "LXNAVIGATION,LX8000F",
+                "gps_receiver": "uBLOX LEA-4S-2,16,max9000m",
+                "pressure_sensor": "INTERSEMA,MS5534A,max10000m",
+                "competition_id": "2H",
+                "competition_class": "Doubleseater",
+            }
+        )
 
-        points = db.session.query(AircraftBeacon) \
-            .filter(AircraftBeacon.device_id == device_id) \
-            .filter(AircraftBeacon.timestamp > date + ' 00:00:00') \
-            .filter(AircraftBeacon.timestamp < date + ' 23:59:59') \
+        points = (
+            db.session.query(AircraftBeacon)
+            .filter(AircraftBeacon.device_id == device_id)
+            .filter(AircraftBeacon.timestamp > date + " 00:00:00")
+            .filter(AircraftBeacon.timestamp < date + " 23:59:59")
             .order_by(AircraftBeacon.timestamp)
+        )
 
         for point in points.all():
-            writer.write_fix(
-                point.timestamp.time(),
-                latitude=point.location.latitude,
-                longitude=point.location.longitude,
-                valid=True,
-                pressure_alt=point.altitude,
-                gps_alt=point.altitude,
-            )
+            writer.write_fix(point.timestamp.time(), latitude=point.location.latitude, longitude=point.location.longitude, valid=True, pressure_alt=point.altitude, gps_alt=point.altitude)

app/commands/flights.py

@@ -4,15 +4,15 @@ import click
 from datetime import datetime
 from tqdm import tqdm
 
-from ogn_python.commands.database import get_database_days
-from ogn_python import db
+from app.commands.database import get_database_days
+from app import db
 
-user_cli = AppGroup('flights')
+user_cli = AppGroup("flights")
 user_cli.help = "Create 2D flight paths from data."
 
-NOTHING = ''
-CONTEST_RELEVANT = 'AND agl < 1000'
-LOW_PASS = 'AND agl < 50 and ground_speed > 250'
+NOTHING = ""
+CONTEST_RELEVANT = "AND agl < 1000"
+LOW_PASS = "AND agl < 50 and ground_speed > 250"
 
 
 def compute_gaps(session, date):
@@ -46,7 +46,9 @@ def compute_gaps(session, date):
             ) sq3
         GROUP BY sq3.device_id
         ON CONFLICT DO NOTHING;
-    """.format(date=date.strftime('%Y-%m-%d'))
+    """.format(
+        date=date.strftime("%Y-%m-%d")
+    )
 
     session.execute(query)
     session.commit()
@@ -111,17 +113,17 @@ def compute_flights2d(session, date, flight_type):
     ) sq5
     GROUP BY sq5.device_id
     ON CONFLICT DO NOTHING;
-    """.format(date=date.strftime('%Y-%m-%d'),
-               flight_type=flight_type,
-               filter=filter)
+    """.format(
+        date=date.strftime("%Y-%m-%d"), flight_type=flight_type, filter=filter
+    )
     session.execute(query)
     session.commit()
 
 
-@user_cli.command('create')
-@click.argument('start')
-@click.argument('end')
-@click.argument('flight_type', type=click.INT)
+@user_cli.command("create")
+@click.argument("start")
+@click.argument("end")
+@click.argument("flight_type", type=click.INT)
 def create(start, end, flight_type):
     """Compute flights. Flight type: 0: all flights, 1: below 1000m AGL, 2: below 50m AGL + faster than 250 km/h, 3: inverse coverage'"""
 
@@ -129,7 +131,7 @@ def create(start, end, flight_type):
 
     pbar = tqdm(days)
     for single_date in pbar:
-        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
+        pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d"))
         if flight_type <= 2:
             result = compute_flights2d(session=db.session, date=single_date, flight_type=flight_type)
         else:

app/commands/gateway.py

@@ -2,33 +2,33 @@ from flask.cli import AppGroup
 import click
 
 from ogn.client import AprsClient
-from ogn_python.gateway.bulkimport import ContinuousDbFeeder
+from app.gateway.bulkimport import ContinuousDbFeeder
 
-from ogn_python import app
+from app import app
 
-user_cli = AppGroup('gateway')
+user_cli = AppGroup("gateway")
 user_cli.help = "Connection to APRS servers."
 
 
-@user_cli.command('run')
-def run(aprs_user='anon-dev'):
+@user_cli.command("run")
+def run(aprs_user="anon-dev"):
     """Run the aprs client."""
 
     saver = ContinuousDbFeeder()
 
     # User input validation
     if len(aprs_user) < 3 or len(aprs_user) > 9:
-        print('aprs_user must be a string of 3-9 characters.')
+        print("aprs_user must be a string of 3-9 characters.")
         return
 
-    app.logger.warning('Start ogn gateway')
+    app.logger.warning("Start ogn gateway")
     client = AprsClient(aprs_user)
     client.connect()
 
     try:
         client.run(callback=saver.add, autoreconnect=True)
     except KeyboardInterrupt:
-        app.logger.warning('\nStop ogn gateway')
+        app.logger.warning("\nStop ogn gateway")
 
     saver.flush()
     client.disconnect()

app/commands/logbook.py

@@ -0,0 +1,119 @@
+from flask.cli import AppGroup
+import click
+
+from datetime import datetime
+
+from app.collect.logbook import update_entries as logbook_update_entries
+from app.collect.takeoff_landings import update_entries as takeoff_landings_update_entries
+from app.model import Airport, Logbook
+from sqlalchemy.sql import func
+from tqdm import tqdm
+from app.commands.database import get_database_days
+from app.utils import date_to_timestamps
+
+from app import db
+
+user_cli = AppGroup("logbook")
+user_cli.help = "Handling of logbook data."
+
+
+@user_cli.command("compute_takeoff_landing")
+@click.argument("start")
+@click.argument("end")
+def compute_takeoff_landing(start, end):
+    """Compute takeoffs and landings."""
+
+    days = get_database_days(start, end)
+
+    pbar = tqdm(days)
+    for single_date in pbar:
+        pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d"))
+        (start, end) = date_to_timestamps(single_date)
+        result = takeoff_landings_update_entries(session=db.session, start=start, end=end)
+
+
+@user_cli.command("compute_logbook")
+@click.argument("start")
+@click.argument("end")
+def compute_logbook(start, end):
+    """Compute logbook."""
+
+    days = get_database_days(start, end)
+
+    pbar = tqdm(days)
+    for single_date in pbar:
+        pbar.set_description(single_date.strftime("%Y-%m-%d"))
+        result = logbook_update_entries(session=db.session, date=single_date)
+
+
+@user_cli.command("show")
+@click.argument("airport_name")
+@click.argument("date")
+def show(airport_name, date=None):
+    """Show a logbook for <airport_name>."""
+    airport = db.session.query(Airport).filter(Airport.name == airport_name).first()
+
+    if airport is None:
+        print('Airport "{}" not found.'.format(airport_name))
+        return
+
+    or_args = []
+    if date is not None:
+        date = datetime.strptime(date, "%Y-%m-%d")
+        (start, end) = date_to_timestamps(date)
+        or_args = [db.between(Logbook.reftime, start, end)]
+
+    # get all logbook entries and add device and airport infos
+    logbook_query = (
+        db.session.query(func.row_number().over(order_by=Logbook.reftime).label("row_number"), Logbook)
+        .filter(*or_args)
+        .filter(db.or_(Logbook.takeoff_airport_id == airport.id, Logbook.landing_airport_id == airport.id))
+        .order_by(Logbook.reftime)
+    )
+
+    # ... and finally print out the logbook
+    print("--- Logbook ({}) ---".format(airport_name))
+
+    def none_datetime_replacer(datetime_object):
+        return "--:--:--" if datetime_object is None else datetime_object.time()
+
+    def none_track_replacer(track_object):
+        return "--" if track_object is None else round(track_object / 10.0)
+
+    def none_timedelta_replacer(timedelta_object):
+        return "--:--:--" if timedelta_object is None else timedelta_object
+
+    def none_registration_replacer(device_object):
+        return "[" + device_object.address + "]" if len(device_object.infos) == 0 else device_object.infos[0].registration
+
+    def none_aircraft_replacer(device_object):
+        return "(unknown)" if len(device_object.infos) == 0 else device_object.infos[0].aircraft
+
+    def airport_marker(logbook_object):
+        if logbook_object.takeoff_airport is not None and logbook_object.takeoff_airport.name is not airport.name:
+            return "FROM: {}".format(logbook_object.takeoff_airport.name)
+        elif logbook_object.landing_airport is not None and logbook_object.landing_airport.name is not airport.name:
+            return "TO: {}".format(logbook_object.landing_airport.name)
+        else:
+            return ""
+
+    def none_altitude_replacer(logbook_object):
+        return "?" if logbook_object.max_altitude is None else "{:5d}m ({:+5d}m)".format(logbook_object.max_altitude, logbook_object.max_altitude - logbook_object.takeoff_airport.altitude)
+
+    for [row_number, logbook] in logbook_query.all():
+        print(
+            "%3d. %10s   %8s (%2s)   %8s (%2s)   %8s  %15s %8s   %17s %20s"
+            % (
+                row_number,
+                logbook.reftime.date(),
+                none_datetime_replacer(logbook.takeoff_timestamp),
+                none_track_replacer(logbook.takeoff_track),
+                none_datetime_replacer(logbook.landing_timestamp),
+                none_track_replacer(logbook.landing_track),
+                none_timedelta_replacer(logbook.duration),
+                none_altitude_replacer(logbook),
+                none_registration_replacer(logbook.device),
+                none_aircraft_replacer(logbook.device),
+                airport_marker(logbook),
+            )
+        )

app/commands/stats.py

@@ -4,24 +4,31 @@ import click
 from datetime import datetime
 from tqdm import tqdm
 
-from ogn_python.commands.database import get_database_days
+from app.commands.database import get_database_days
 
-from ogn_python.collect.stats import create_device_stats, create_receiver_stats, create_relation_stats, create_country_stats,\
-    update_qualities, update_receivers as update_receivers_command, update_devices as update_devices_command,\
-    update_device_stats_jumps
+from app.collect.stats import (
+    create_device_stats,
+    create_receiver_stats,
+    create_relation_stats,
+    create_country_stats,
+    update_qualities,
+    update_receivers as update_receivers_command,
+    update_devices as update_devices_command,
+    update_device_stats_jumps,
+)
 
-from ogn_python.collect.ognrange import update_entries as update_receiver_coverages
+from app.collect.ognrange import update_entries as update_receiver_coverages
 
-from ogn_python import db
+from app import db
 
 
-user_cli = AppGroup('stats')
+user_cli = AppGroup("stats")
 user_cli.help = "Handling of statistical data."
 
 
-@user_cli.command('create')
-@click.argument('start')
-@click.argument('end')
+@user_cli.command("create")
+@click.argument("start")
+@click.argument("end")
 def create(start, end):
     """Create DeviceStats, ReceiverStats and RelationStats."""
 
@@ -29,16 +36,17 @@ def create(start, end):
 
     pbar = tqdm(days)
     for single_date in pbar:
-        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
+        pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d"))
         result = create_device_stats(session=db.session, date=single_date)
         result = update_device_stats_jumps(session=db.session, date=single_date)
         result = create_receiver_stats(session=db.session, date=single_date)
         result = create_relation_stats(session=db.session, date=single_date)
         result = update_qualities(session=db.session, date=single_date)
 
-@user_cli.command('create_country')
-@click.argument('start')
-@click.argument('end')
+
+@user_cli.command("create_country")
+@click.argument("start")
+@click.argument("end")
 def create_country(start, end):
     """Create CountryStats."""
 
@@ -46,22 +54,29 @@ def create_country(start, end):
 
     pbar = tqdm(days)
     for single_date in pbar:
-        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
+        pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d"))
         result = create_country_stats(session=db.session, date=single_date)
 
-from ogn_python.model import *
-@user_cli.command('update_devices_name')
+
+from app.model import *
+
+
+@user_cli.command("update_devices_name")
 def update_devices_name():
     """Update Devices name."""
 
     device_ids = db.session.query(Device.id).all()
 
     for device_id in tqdm(device_ids):
-        db.session.execute("update devices d set name = sq.name from ( select * from aircraft_beacons ab where ab.device_id = {} limit 1) sq where d.id = sq.device_id and d.name is null or d.name = 'ICA3D3CC4';".format(device_id[0]))
+        db.session.execute(
+            "update devices d set name = sq.name from ( select * from aircraft_beacons ab where ab.device_id = {} limit 1) sq where d.id = sq.device_id and d.name is null or d.name = 'ICA3D3CC4';".format(
+                device_id[0]
+            )
+        )
         db.session.commit()
 
 
-@user_cli.command('update_receivers')
+@user_cli.command("update_receivers")
 def update_receivers():
     """Update receivers with data from stats."""
 
@@ -69,7 +84,7 @@ def update_receivers():
     print(result)
 
 
-@user_cli.command('update_devices')
+@user_cli.command("update_devices")
 def update_devices():
     """Update devices with data from stats."""
 
@@ -77,9 +92,9 @@ def update_devices():
     print(result)
 
 
-@user_cli.command('update_mgrs')
-@click.argument('start')
-@click.argument('end')
+@user_cli.command("update_mgrs")
+@click.argument("start")
+@click.argument("end")
 def update_mgrs(start, end):
     """Create location_mgrs_short."""
 
@@ -87,23 +102,25 @@ def update_mgrs(start, end):
 
     pbar = tqdm(days)
     for single_date in pbar:
-        datestr = datetime.strftime(single_date, '%Y-%m-%d')
+        datestr = datetime.strftime(single_date, "%Y-%m-%d")
         pbar.set_description(datestr)
         for pbar2 in tqdm(["{:02d}:{:02d}".format(hh, mm) for hh in range(0, 24) for mm in range(0, 60)]):
             sql = """
                 UPDATE aircraft_beacons
                 SET location_mgrs_short = left(location_mgrs, 5) || substring(location_mgrs, 6, 2) || substring(location_mgrs, 11, 2)
                 WHERE timestamp BETWEEN '{0} {1}:00' and '{0} {1}:59' AND location_mgrs_short IS NULL;
-            """.format(datestr, pbar2)
+            """.format(
+                datestr, pbar2
+            )
 
-            #print(sql)
+            # print(sql)
             db.session.execute(sql)
             db.session.commit()
 
 
-@user_cli.command('create_ognrange')
-@click.argument('start')
-@click.argument('end')
+@user_cli.command("create_ognrange")
+@click.argument("start")
+@click.argument("end")
 def create_ognrange(start=None, end=None):
     """Create receiver coverage stats for Melissas ognrange."""
 
@@ -111,5 +128,5 @@ def create_ognrange(start=None, end=None):
 
     pbar = tqdm(days)
     for single_date in pbar:
-        pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
+        pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d"))
         result = update_receiver_coverages(session=db.session, date=single_date)

app/config/default.py

@@ -0,0 +1,27 @@
+SECRET_KEY = "i-like-ogn"
+
+SQLALCHEMY_DATABASE_URI = "postgresql://postgres@localhost:5432/ogn"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+
+# Flask-Cache stuff
+CACHE_TYPE = "simple"
+CACHE_DEFAULT_TIMEOUT = 300
+
+# Celery stuff
+CELERY_BROKER_URL = "redis://localhost:6379/0"
+CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+
+
+from celery.schedules import crontab
+from datetime import timedelta
+
+CELERYBEAT_SCHEDULE = {
+    "update-ddb": {"task": "import_ddb", "schedule": timedelta(hours=1)},
+    "update-country-codes": {"task": "update_receivers_country_code", "schedule": timedelta(days=1)},
+    "update-takeoff-and-landing": {"task": "update_takeoff_landings", "schedule": timedelta(hours=1), "kwargs": {"last_minutes": 90}},
+    "update-logbook": {"task": "update_logbook_entries", "schedule": timedelta(hours=2), "kwargs": {"day_offset": 0}},
+    "update-max-altitudes": {"task": "update_logbook_max_altitude", "schedule": timedelta(hours=1), "kwargs": {"day_offset": 0}},
+    "update-stats-daily": {"task": "update_stats", "schedule": crontab(hour=0, minute=5), "kwargs": {"day_offset": -1}},
+    "update-logbook-daily": {"task": "update_logbook_entries", "schedule": crontab(hour=1, minute=0), "kwargs": {"day_offset": -1}},
+    "purge_old_data": {"task": "purge_old_data", "schedule": timedelta(hours=1), "kwargs": {"max_hours": 48}},
+}

app/config/test.py

@@ -0,0 +1,6 @@
+SQLALCHEMY_DATABASE_URI = "postgresql://postgres@localhost:5432/ogn_test"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+
+# Celery stuff
+CELERY_BROKER_URL = "redis://localhost:6379/0"
+CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

app/flask_celery.py

@@ -1,11 +1,8 @@
 from celery import Celery
 
+
 def make_celery(app):
-    celery = Celery(
-        app.import_name,
-        backend=app.config['CELERY_RESULT_BACKEND'],
-        broker=app.config['CELERY_BROKER_URL']
-    )
+    celery = Celery(app.import_name, backend=app.config["CELERY_RESULT_BACKEND"], broker=app.config["CELERY_BROKER_URL"])
     celery.conf.update(app.config)
 
     class ContextTask(celery.Task):
@@ -14,4 +11,4 @@ def make_celery(app):
                 return self.run(*args, **kwargs)
 
     celery.Task = ContextTask
-    return celery
+    return celery

app/gateway/bulkimport.py

@@ -0,0 +1,407 @@
+from datetime import datetime, timedelta
+from io import StringIO
+
+from flask.cli import AppGroup
+import click
+from tqdm import tqdm
+from mgrs import MGRS
+
+from ogn.parser import parse, ParseError
+
+from app.model import AircraftBeacon, ReceiverBeacon, Location
+from app.utils import open_file
+from app.gateway.process_tools import *
+
+from app import db
+from app import app
+
+user_cli = AppGroup("bulkimport")
+user_cli.help = "Tools for accelerated data import."
+
+
+# define message types we want to proceed
+AIRCRAFT_BEACON_TYPES = ["aprs_aircraft", "flarm", "tracker", "fanet", "lt24", "naviter", "skylines", "spider", "spot"]
+RECEIVER_BEACON_TYPES = ["aprs_receiver", "receiver"]
+
+# define fields we want to proceed
+BEACON_KEY_FIELDS = ["name", "receiver_name", "timestamp"]
+AIRCRAFT_BEACON_FIELDS = [
+    "location",
+    "altitude",
+    "dstcall",
+    "relay",
+    "track",
+    "ground_speed",
+    "address_type",
+    "aircraft_type",
+    "stealth",
+    "address",
+    "climb_rate",
+    "turn_rate",
+    "signal_quality",
+    "error_count",
+    "frequency_offset",
+    "gps_quality_horizontal",
+    "gps_quality_vertical",
+    "software_version",
+    "hardware_version",
+    "real_address",
+    "signal_power",
+    "distance",
+    "radial",
+    "quality",
+    "location_mgrs",
+    "location_mgrs_short",
+    "agl",
+    "receiver_id",
+    "device_id",
+]
+RECEIVER_BEACON_FIELDS = [
+    "location",
+    "altitude",
+    "dstcall",
+    "relay",
+    "version",
+    "platform",
+    "cpu_load",
+    "free_ram",
+    "total_ram",
+    "ntp_error",
+    "rt_crystal_correction",
+    "voltage",
+    "amperage",
+    "cpu_temp",
+    "senders_visible",
+    "senders_total",
+    "rec_input_noise",
+    "senders_signal",
+    "senders_messages",
+    "good_senders_signal",
+    "good_senders",
+    "good_and_bad_senders",
+]
+
+
+myMGRS = MGRS()
+
+
+def string_to_message(raw_string, reference_date):
+    global receivers
+
+    try:
+        message = parse(raw_string, reference_date)
+    except NotImplementedError as e:
+        app.logger.error("No parser implemented for message: {}".format(raw_string))
+        return None
+    except ParseError as e:
+        app.logger.error("Parsing error with message: {}".format(raw_string))
+        return None
+    except TypeError as e:
+        app.logger.error("TypeError with message: {}".format(raw_string))
+        return None
+    except Exception as e:
+        app.logger.error("Other Exception with string: {}".format(raw_string))
+        return None
+
+    # update reference receivers and distance to the receiver
+    if message["aprs_type"] == "position":
+        if message["beacon_type"] in AIRCRAFT_BEACON_TYPES + RECEIVER_BEACON_TYPES:
+            latitude = message["latitude"]
+            longitude = message["longitude"]
+
+            location = Location(longitude, latitude)
+            message["location"] = location.to_wkt()
+            location_mgrs = myMGRS.toMGRS(latitude, longitude).decode("utf-8")
+            message["location_mgrs"] = location_mgrs
+            message["location_mgrs_short"] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12]
+
+        if message["beacon_type"] in AIRCRAFT_BEACON_TYPES and "gps_quality" in message:
+            if message["gps_quality"] is not None and "horizontal" in message["gps_quality"]:
+                message["gps_quality_horizontal"] = message["gps_quality"]["horizontal"]
+                message["gps_quality_vertical"] = message["gps_quality"]["vertical"]
+            del message["gps_quality"]
+
+    # TODO: Fix python-ogn-client 0.91
+    if "senders_messages" in message and message["senders_messages"] is not None:
+        message["senders_messages"] = int(message["senders_messages"])
+    if "good_senders" in message and message["good_senders"] is not None:
+        message["good_senders"] = int(message["good_senders"])
+    if "good_and_bad_senders" in message and message["good_and_bad_senders"] is not None:
+        message["good_and_bad_senders"] = int(message["good_and_bad_senders"])
+
+    return message
+
+
+class ContinuousDbFeeder:
+    def __init__(self,):
+        self.postfix = "continuous_import"
+        self.last_flush = datetime.utcnow()
+        self.last_add_missing = datetime.utcnow()
+        self.last_transfer = datetime.utcnow()
+
+        self.aircraft_buffer = StringIO()
+        self.receiver_buffer = StringIO()
+
+        create_tables(self.postfix)
+        create_indices(self.postfix)
+
+    def add(self, raw_string):
+        message = string_to_message(raw_string, reference_date=datetime.utcnow())
+
+        if message is None or ("raw_message" in message and message["raw_message"][0] == "#") or "beacon_type" not in message:
+            return
+
+        if message["beacon_type"] in AIRCRAFT_BEACON_TYPES:
+            complete_message = ",".join([str(message[k]) if k in message and message[k] is not None else "\\N" for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS])
+            self.aircraft_buffer.write(complete_message)
+            self.aircraft_buffer.write("\n")
+        elif message["beacon_type"] in RECEIVER_BEACON_TYPES:
+            complete_message = ",".join([str(message[k]) if k in message and message[k] is not None else "\\N" for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS])
+            self.receiver_buffer.write(complete_message)
+            self.receiver_buffer.write("\n")
+        else:
+            app.logger.error("Ignore beacon_type: {}".format(message["beacon_type"]))
+            return
+
+        if datetime.utcnow() - self.last_flush >= timedelta(seconds=20):
+            self.flush()
+            self.prepare()
+
+            self.aircraft_buffer = StringIO()
+            self.receiver_buffer = StringIO()
+
+            self.last_flush = datetime.utcnow()
+
+        if datetime.utcnow() - self.last_add_missing >= timedelta(seconds=60):
+            self.add_missing()
+            self.last_add_missing = datetime.utcnow()
+
+        if datetime.utcnow() - self.last_transfer >= timedelta(seconds=30):
+            self.transfer()
+            self.delete_beacons()
+            self.last_transfer = datetime.utcnow()
+
+    def flush(self):
+        self.aircraft_buffer.seek(0)
+        self.receiver_buffer.seek(0)
+
+        connection = db.engine.raw_connection()
+        cursor = connection.cursor()
+        cursor.copy_from(self.aircraft_buffer, "aircraft_beacons_{0}".format(self.postfix), sep=",", columns=BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS)
+        cursor.copy_from(self.receiver_buffer, "receiver_beacons_{0}".format(self.postfix), sep=",", columns=BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS)
+        connection.commit()
+
+        self.aircraft_buffer = StringIO()
+        self.receiver_buffer = StringIO()
+
+    def add_missing(self):
+        add_missing_receivers(self.postfix)
+        add_missing_devices(self.postfix)
+
+    def prepare(self):
+        # make receivers complete
+        update_receiver_beacons(self.postfix)
+        update_receiver_location(self.postfix)
+
+        # make devices complete
+        update_aircraft_beacons(self.postfix)
+
+    def transfer(self):
+        # tranfer beacons
+        transfer_aircraft_beacons(self.postfix)
+        transfer_receiver_beacons(self.postfix)
+
+    def delete_beacons(self):
+        # delete already transfered beacons
+        delete_receiver_beacons(self.postfix)
+        delete_aircraft_beacons(self.postfix)
+
+
+class FileDbFeeder:
+    def __init__(self):
+        self.postfix = "continuous_import"
+        self.last_flush = datetime.utcnow()
+
+        self.aircraft_buffer = StringIO()
+        self.receiver_buffer = StringIO()
+
+        create_tables(self.postfix)
+        create_indices(self.postfix)
+
+    def add(self, raw_string):
+        message = string_to_message(raw_string, reference_date=datetime.utcnow())
+
+        if message is None or ("raw_message" in message and message["raw_message"][0] == "#") or "beacon_type" not in message:
+            return
+
+        if message["beacon_type"] in AIRCRAFT_BEACON_TYPES:
+            complete_message = ",".join([str(message[k]) if k in message and message[k] is not None else "\\N" for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS])
+            self.aircraft_buffer.write(complete_message)
+            self.aircraft_buffer.write("\n")
+        elif message["beacon_type"] in RECEIVER_BEACON_TYPES:
+            complete_message = ",".join([str(message[k]) if k in message and message[k] is not None else "\\N" for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS])
+            self.receiver_buffer.write(complete_message)
+            self.receiver_buffer.write("\n")
+        else:
+            app.logger.error("Ignore beacon_type: {}".format(message["beacon_type"]))
+            return
+
+    def prepare(self):
+        # make receivers complete
+        add_missing_receivers(self.postfix)
+        update_receiver_location(self.postfix)
+
+        # make devices complete
+        add_missing_devices(self.postfix)
+
+        # prepare beacons for transfer
+        create_indices(self.postfix)
+        update_receiver_beacons_bigdata(self.postfix)
+        update_aircraft_beacons_bigdata(self.postfix)
+
+
+def get_aircraft_beacons_postfixes():
+    """Get the postfixes from imported aircraft_beacons logs."""
+
+    postfixes = db.session.execute(
+        """
+        SELECT DISTINCT(RIGHT(tablename, 8))
+        FROM pg_catalog.pg_tables
+        WHERE schemaname = 'public' AND tablename LIKE 'aircraft\_beacons\_20______'
+        ORDER BY RIGHT(tablename, 10);
+    """
+    ).fetchall()
+
+    return [postfix for postfix in postfixes]
+
+
+def export_to_path(postfix):
+    import os, gzip
+
+    aircraft_beacons_file = os.path.join(path, "aircraft_beacons_{0}.csv.gz".format(postfix))
+    with gzip.open(aircraft_beacons_file, "wt", encoding="utf-8") as gzip_file:
+        self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_aircraft_beacons_subquery()), gzip_file)
+    receiver_beacons_file = os.path.join(path, "receiver_beacons_{0}.csv.gz".format(postfix))
+    with gzip.open(receiver_beacons_file, "wt") as gzip_file:
+        self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_receiver_beacons_subquery()), gzip_file)
+
+
+def convert(sourcefile, datestr, saver):
+    from app.gateway.process import string_to_message
+    from app.gateway.process_tools import AIRCRAFT_BEACON_TYPES, RECEIVER_BEACON_TYPES
+    from datetime import datetime
+
+    fin = open_file(sourcefile)
+
+    # get total lines of the input file
+    total_lines = 0
+    for line in fin:
+        total_lines += 1
+    fin.seek(0)
+
+    current_line = 0
+    steps = 100000
+    reference_date = datetime.strptime(datestr + " 12:00:00", "%Y-%m-%d %H:%M:%S")
+
+    pbar = tqdm(fin, total=total_lines)
+    for line in pbar:
+        pbar.set_description("Importing {}".format(sourcefile))
+
+        current_line += 1
+        if current_line % steps == 0:
+            saver.flush()
+
+        message = string_to_message(line.strip(), reference_date=reference_date)
+        if message is None:
+            continue
+
+        dictfilt = lambda x, y: dict([(i, x[i]) for i in x if i in set(y)])
+
+        try:
+            if message["beacon_type"] in AIRCRAFT_BEACON_TYPES:
+                message = dictfilt(
+                    message,
+                    (
+                        "beacon_type",
+                        "aprs_type",
+                        "location_wkt",
+                        "altitude",
+                        "name",
+                        "dstcall",
+                        "relay",
+                        "receiver_name",
+                        "timestamp",
+                        "track",
+                        "ground_speed",
+                        "address_type",
+                        "aircraft_type",
+                        "stealth",
+                        "address",
+                        "climb_rate",
+                        "turn_rate",
+                        "signal_quality",
+                        "error_count",
+                        "frequency_offset",
+                        "gps_quality_horizontal",
+                        "gps_quality_vertical",
+                        "software_version",
+                        "hardware_version",
+                        "real_address",
+                        "signal_power",
+                        "distance",
+                        "radial",
+                        "quality",
+                        "agl",
+                        "location_mgrs",
+                        "location_mgrs_short",
+                        "receiver_id",
+                        "device_id",
+                    ),
+                )
+
+                beacon = AircraftBeacon(**message)
+            elif message["beacon_type"] in RECEIVER_BEACON_TYPES:
+                if "rec_crystal_correction" in message:
+                    del message["rec_crystal_correction"]
+                    del message["rec_crystal_correction_fine"]
+                beacon = ReceiverBeacon(**message)
+            saver.add(beacon)
+        except Exception as e:
+            print(e)
+
+    saver.flush()
+    fin.close()
+
+
+@user_cli.command("file_import")
+@click.argument("path")
+def file_import(path):
+    """Import APRS logfiles into separate logfile tables."""
+
+    import os
+    import re
+
+    # Get Filepaths and dates to import
+    results = list()
+    for (root, dirs, files) in os.walk(path):
+        for file in sorted(files):
+            match = re.match("OGN_log\.txt_([0-9]{4}\-[0-9]{2}\-[0-9]{2})\.gz$", file)
+            if match:
+                results.append({"filepath": os.path.join(root, file), "datestr": match.group(1)})
+
+    with LogfileDbSaver() as saver:
+        already_imported = saver.get_datestrs()
+
+        results = list(filter(lambda x: x["datestr"] not in already_imported, results))
+
+        pbar = tqdm(results)
+        for result in pbar:
+            filepath = result["filepath"]
+            datestr = result["datestr"]
+            pbar.set_description("Importing data for {}".format(datestr))
+
+            saver.set_datestr(datestr)
+            saver.create_tables()
+            convert(filepath, datestr, saver)
+            saver.add_missing_devices()
+            saver.add_missing_receivers()

app/gateway/process_tools.py

@@ -1,4 +1,5 @@
-from ogn_python import db
+from app import db
+
 
 def create_tables(postfix):
     """Create tables for log file import."""
@@ -11,42 +12,55 @@ def create_tables(postfix):
 def create_indices(postfix):
     """Creates indices for aircraft- and receiver-beacons."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_device_id ON "aircraft_beacons_{0}" (device_id NULLS FIRST);
         CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_receiver_id ON "aircraft_beacons_{0}" (receiver_id NULLS FIRST);
         CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_timestamp_name_receiver_name ON "aircraft_beacons_{0}" (timestamp, name, receiver_name);
         CREATE INDEX IF NOT EXISTS ix_receiver_beacons_{0}_timestamp_name_receiver_name ON "receiver_beacons_{0}" (timestamp, name, receiver_name);
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def create_indices_bigdata(postfix):
     """Creates indices for aircraft- and receiver-beacons."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_timestamp_name_receiver_name ON "aircraft_beacons_{0}" (timestamp, name, receiver_name);
         CREATE INDEX IF NOT EXISTS ix_receiver_beacons_{0}_timestamp_name_receiver_name ON "receiver_beacons_{0}" (timestamp, name, receiver_name);
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def add_missing_devices(postfix):
     """Add missing devices."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         INSERT INTO devices(address)
         SELECT DISTINCT (ab.address)
         FROM "aircraft_beacons_{0}" AS ab
         WHERE ab.address IS NOT NULL AND NOT EXISTS (SELECT 1 FROM devices AS d WHERE d.address = ab.address)
         ORDER BY ab.address;
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def add_missing_receivers(postfix):
     """Add missing receivers."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         INSERT INTO receivers(name)
         SELECT DISTINCT (rb.name)
         FROM "receiver_beacons_{0}" AS rb
@@ -58,14 +72,18 @@ def add_missing_receivers(postfix):
         FROM "aircraft_beacons_{0}" AS ab
         WHERE NOT EXISTS (SELECT 1 FROM receivers AS r WHERE r.name = ab.receiver_name)
         ORDER BY ab.receiver_name;
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def update_receiver_location(postfix):
     """Updates the receiver location. We need this because we want the actual location for distance calculations."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         UPDATE receivers AS r
         SET
             location = sq.location,
@@ -77,19 +95,26 @@ def update_receiver_location(postfix):
             ORDER BY rb.receiver_id, rb.timestamp
             ) AS sq
         WHERE r.id = sq.receiver_id;
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def update_receiver_beacons(postfix):
     """Updates the foreign keys."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         UPDATE receiver_beacons_{0} AS rb
         SET receiver_id = r.id
         FROM receivers AS r
         WHERE rb.receiver_id IS NULL AND rb.name = r.name;
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
@@ -97,7 +122,8 @@ def update_receiver_beacons_bigdata(postfix):
     """Updates the foreign keys.
        Due to performance reasons we use a new table instead of updating the old."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         SELECT
             rb.location, rb.altitude, rb.name, rb.receiver_name, rb.dstcall, rb.timestamp,
 
@@ -112,7 +138,10 @@ def update_receiver_beacons_bigdata(postfix):
 
         DROP TABLE IF EXISTS "receiver_beacons_{0}";
         ALTER TABLE "receiver_beacons_{0}_temp" RENAME TO "receiver_beacons_{0}";
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
@@ -120,7 +149,8 @@ def update_aircraft_beacons(postfix):
     """Updates the foreign keys and calculates distance/radial and quality and computes the altitude above ground level.
        Elevation data has to be in the table 'elevation' with srid 4326."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         UPDATE aircraft_beacons_{0} AS ab
         SET
             device_id = d.id,
@@ -135,7 +165,10 @@ def update_aircraft_beacons(postfix):
 
         FROM devices AS d, receivers AS r, elevation AS e
         WHERE ab.device_id IS NULL and ab.receiver_id IS NULL AND ab.address = d.address AND ab.receiver_name = r.name AND ST_Intersects(e.rast, ab.location);
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
@@ -144,7 +177,8 @@ def update_aircraft_beacons_bigdata(postfix):
        Elevation data has to be in the table 'elevation' with srid 4326.
        Due to performance reasons we use a new table instead of updating the old."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         SELECT
             ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed,
 
@@ -170,14 +204,18 @@ def update_aircraft_beacons_bigdata(postfix):
 
         DROP TABLE IF EXISTS "aircraft_beacons_{0}";
         ALTER TABLE "aircraft_beacons_{0}_temp" RENAME TO "aircraft_beacons_{0}";
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def delete_receiver_beacons(postfix):
     """Delete beacons from table."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         DELETE FROM receiver_beacons_continuous_import AS rb
         USING (
             SELECT name, receiver_name, timestamp
@@ -185,14 +223,18 @@ def delete_receiver_beacons(postfix):
             WHERE receiver_id IS NOT NULL
         ) AS sq
         WHERE rb.name = sq.name AND rb.receiver_name = sq.receiver_name AND rb.timestamp = sq.timestamp
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
 def delete_aircraft_beacons(postfix):
     """Delete beacons from table."""
 
-    db.session.execute("""
+    db.session.execute(
+        """
         DELETE FROM aircraft_beacons_continuous_import AS ab
         USING (
             SELECT name, receiver_name, timestamp
@@ -200,7 +242,10 @@ def delete_aircraft_beacons(postfix):
             WHERE receiver_id IS NOT NULL and device_id IS NOT NULL
         ) AS sq
         WHERE ab.name = sq.name AND ab.receiver_name = sq.receiver_name AND ab.timestamp = sq.timestamp
-    """.format(postfix))
+    """.format(
+            postfix
+        )
+    )
     db.session.commit()
 
 
@@ -247,7 +292,9 @@ def get_merged_aircraft_beacons_subquery(postfix):
     FROM "aircraft_beacons_{0}" AS ab
     GROUP BY timestamp, name, receiver_name
     ORDER BY timestamp, name, receiver_name
-    """.format(postfix)
+    """.format(
+        postfix
+    )
 
 
 def get_merged_receiver_beacons_subquery(postfix):
@@ -285,7 +332,9 @@ def get_merged_receiver_beacons_subquery(postfix):
     FROM "receiver_beacons_{0}" AS rb
     GROUP BY timestamp, name, receiver_name
     ORDER BY timestamp, name, receiver_name
-    """.format(postfix)
+    """.format(
+        postfix
+    )
 
 
 def transfer_aircraft_beacons(postfix):
@@ -298,7 +347,9 @@ def transfer_aircraft_beacons(postfix):
     FROM ({}) sq
     WHERE sq.receiver_id IS NOT NULL AND sq.device_id IS NOT NULL
     ON CONFLICT DO NOTHING;
-    """.format(get_merged_aircraft_beacons_subquery(postfix))
+    """.format(
+        get_merged_aircraft_beacons_subquery(postfix)
+    )
 
     db.session.execute(query)
     db.session.commit()
@@ -317,7 +368,9 @@ def transfer_receiver_beacons(postfix):
     FROM ({}) sq
     WHERE sq.receiver_id IS NOT NULL
     ON CONFLICT DO NOTHING;
-    """.format(get_merged_receiver_beacons_subquery(postfix))
+    """.format(
+        get_merged_receiver_beacons_subquery(postfix)
+    )
 
     db.session.execute(query)
     db.session.commit()

app/live_routes.py

@@ -0,0 +1,90 @@
+from flask import request, render_template, make_response, send_file
+from flask_cors import cross_origin
+
+from app.backend.liveglidernet import rec, lxml
+
+from app import app
+from app import db
+from app import cache
+
+
+@app.route("/live.html")
+@cross_origin()
+def live():
+    return render_template("ogn_live.html", host=request.host)
+
+
+@app.route("/rec.php")
+def rec_php():
+    a = request.args.get("a")
+    z = request.args.get("z")
+
+    xml = rec()
+    resp = app.make_response(xml)
+    resp.mimetype = "text/xml"
+    return resp
+
+
+@app.route("/lxml.php")
+def lxml_php():
+    a = request.args.get("a")
+    b = request.args.get("b")
+    c = request.args.get("c")
+    d = request.args.get("d")
+    e = request.args.get("e")
+    z = request.args.get("z")
+
+    xml = lxml()
+    resp = app.make_response(xml)
+    resp.mimetype = "text/xml"
+    return resp
+
+
+@app.route("/pict/<filename>")
+def pict(filename):
+    return app.send_static_file("ognlive/pict/" + filename)
+
+
+@app.route("/favicon.gif")
+def favicon_gif():
+    return app.send_static_file("ognlive/pict/favicon.gif")
+
+
+@app.route("/horizZoomControl.js")
+def horizZoomControl_js():
+    return app.send_static_file("ognlive/horizZoomControl.js")
+
+
+@app.route("/barogram.js")
+def barogram_js():
+    return app.send_static_file("ognlive/barogram.js")
+
+
+@app.route("/util.js")
+def util_js():
+    return app.send_static_file("ognlive/util.js")
+
+
+@app.route("/ogn.js")
+def ogn_js():
+    return app.send_static_file("ognlive/ogn.js")
+
+
+@app.route("/ol.js")
+def ol_js():
+    return app.send_static_file("ognlive/ol.js")
+
+
+@app.route("/osm.js")
+def osm_js():
+    return app.send_static_file("ognlive/osm.js")
+
+
+@app.route("/ol.css")
+def ol_css():
+    return app.send_static_file("ognlive/ol.css")
+
+
+@app.route("/osm.css")
+def osm_css():
+    return app.send_static_file("ognlive/osm.css")

app/model/aircraft_beacon.py

@@ -1,7 +1,7 @@
 from sqlalchemy.sql import func
 from .beacon import Beacon
 
-from ogn_python import db
+from app import db
 
 
 class AircraftBeacon(Beacon):
@@ -28,21 +28,21 @@ class AircraftBeacon(Beacon):
     # Calculated values
     distance = db.Column(db.Float(precision=2))
     radial = db.Column(db.SmallInteger)
-    quality = db.Column(db.Float(precision=2))        # signal quality normalized to 10km
-    location_mgrs = db.Column(db.String(15))          # full mgrs (15 chars)
-    location_mgrs_short = db.Column(db.String(9))     # reduced mgrs (9 chars), e.g. used for melissas range tool
+    quality = db.Column(db.Float(precision=2))  # signal quality normalized to 10km
+    location_mgrs = db.Column(db.String(15))  # full mgrs (15 chars)
+    location_mgrs_short = db.Column(db.String(9))  # reduced mgrs (9 chars), e.g. used for melissas range tool
     agl = db.Column(db.Float(precision=2))
 
     # Relations
-    receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'))
-    receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref='aircraft_beacons')
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="SET NULL"))
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref="aircraft_beacons")
 
-    device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'))
-    device = db.relationship('Device', foreign_keys=[device_id], backref='aircraft_beacons')
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="SET NULL"))
+    device = db.relationship("Device", foreign_keys=[device_id], backref="aircraft_beacons")
 
     # Multi-column indices
-    db.Index('ix_aircraft_beacons_receiver_id_distance', 'receiver_id', 'distance')
-    db.Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp')
+    db.Index("ix_aircraft_beacons_receiver_id_distance", "receiver_id", "distance")
+    db.Index("ix_aircraft_beacons_device_id_timestamp", "device_id", "timestamp")
 
     def __repr__(self):
         return "<AircraftBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
@@ -61,49 +61,48 @@ class AircraftBeacon(Beacon):
             self.hardware_version,
             self.real_address,
             self.signal_power,
-
             self.distance,
             self.radial,
             self.quality,
             self.location_mgrs,
-            self.location_mgrs_short)
+            self.location_mgrs_short,
+        )
 
     @classmethod
     def get_columns(self):
-        return ['location',
-                'altitude',
-                'name',
-                'dstcall',
-                'relay',
-                'receiver_name',
-                'timestamp',
-                'track',
-                'ground_speed',
-
-                #'raw_message',
-                #'reference_timestamp',
-
-                'address_type',
-                'aircraft_type',
-                'stealth',
-                'address',
-                'climb_rate',
-                'turn_rate',
-                'signal_quality',
-                'error_count',
-                'frequency_offset',
-                'gps_quality_horizontal',
-                'gps_quality_vertical',
-                'software_version',
-                'hardware_version',
-                'real_address',
-                'signal_power',
-
-                'distance',
-                'radial',
-                'quality',
-                'location_mgrs',
-                'location_mgrs_short']
+        return [
+            "location",
+            "altitude",
+            "name",
+            "dstcall",
+            "relay",
+            "receiver_name",
+            "timestamp",
+            "track",
+            "ground_speed",
+            #'raw_message',
+            #'reference_timestamp',
+            "address_type",
+            "aircraft_type",
+            "stealth",
+            "address",
+            "climb_rate",
+            "turn_rate",
+            "signal_quality",
+            "error_count",
+            "frequency_offset",
+            "gps_quality_horizontal",
+            "gps_quality_vertical",
+            "software_version",
+            "hardware_version",
+            "real_address",
+            "signal_power",
+            "distance",
+            "radial",
+            "quality",
+            "location_mgrs",
+            "location_mgrs_short",
+        ]
 
     def get_values(self):
         return [
@@ -116,10 +115,8 @@ class AircraftBeacon(Beacon):
             self.timestamp,
             self.track,
             self.ground_speed,
-
-            #self.raw_message,
-            #self.reference_timestamp,
-
+            # self.raw_message,
+            # self.reference_timestamp,
             self.address_type,
             self.aircraft_type,
             self.stealth,
@@ -135,13 +132,13 @@ class AircraftBeacon(Beacon):
             self.hardware_version,
             self.real_address,
             self.signal_power,
-
             self.distance,
             self.radial,
             self.quality,
             self.location_mgrs,
-            self.location_mgrs_short]
+            self.location_mgrs_short,
+        ]
 
 
-db.Index('ix_aircraft_beacons_date_device_id_address', func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address)
-db.Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance)
+db.Index("ix_aircraft_beacons_date_device_id_address", func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address)
+db.Index("ix_aircraft_beacons_date_receiver_id_distance", func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance)

app/model/airport.py

@@ -1,6 +1,6 @@
 from geoalchemy2.types import Geometry
 
-from ogn_python import db
+from app import db
 
 
 class Airport(db.Model):
@@ -8,7 +8,7 @@ class Airport(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
 
-    location_wkt = db.Column('location', Geometry('POINT', srid=4326))
+    location_wkt = db.Column("location", Geometry("POINT", srid=4326))
     altitude = db.Column(db.Float(precision=2))
 
     name = db.Column(db.String, index=True)
@@ -20,7 +20,7 @@ class Airport(db.Model):
     runway_length = db.Column(db.SmallInteger)
     frequency = db.Column(db.Float(precision=2))
 
-    border = db.Column('border', Geometry('POLYGON', srid=4326))
+    border = db.Column("border", Geometry("POLYGON", srid=4326))
 
     def __repr__(self):
         return "<Airport %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,% s>" % (
@@ -34,4 +34,5 @@ class Airport(db.Model):
             self.altitude,
             self.runway_direction,
             self.runway_length,
-            self.frequency)
+            self.frequency,
+        )

app/model/beacon.py

@@ -5,12 +5,12 @@ from sqlalchemy.ext.hybrid import hybrid_property
 
 from .geo import Location
 
-from ogn_python import db
+from app import db
 
 
 class Beacon(AbstractConcreteBase, db.Model):
     # APRS data
-    location_wkt = db.Column('location', Geometry('POINT', srid=4326))
+    location_wkt = db.Column("location", Geometry("POINT", srid=4326))
     altitude = db.Column(db.Float(precision=2))
 
     name = db.Column(db.String, primary_key=True, nullable=True)

app/model/country.py

@@ -1,6 +1,6 @@
 from geoalchemy2.types import Geometry
 
-from ogn_python import db
+from app import db
 
 
 class Country(db.Model):
@@ -21,18 +21,7 @@ class Country(db.Model):
     lon = db.Column(db.Float)
     lat = db.Column(db.Float)
 
-    geom = db.Column('geom', Geometry('MULTIPOLYGON', srid=4326))
+    geom = db.Column("geom", Geometry("MULTIPOLYGON", srid=4326))
 
     def __repr__(self):
-        return "<Country %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
-            self.fips,
-            self.iso2,
-            self.iso3,
-            self.un,
-            self.name,
-            self.area,
-            self.pop2005,
-            self.region,
-            self.subregion,
-            self.lon,
-            self.lat)
+        return "<Country %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (self.fips, self.iso2, self.iso3, self.un, self.name, self.area, self.pop2005, self.region, self.subregion, self.lon, self.lat)

app/model/country_stats.py

@@ -1,4 +1,4 @@
-from ogn_python import db
+from app import db
 
 
 class CountryStats(db.Model):
@@ -13,6 +13,5 @@ class CountryStats(db.Model):
     device_count = db.Column(db.Integer)
 
     # Relations
-    country_id = db.Column(db.Integer, db.ForeignKey('countries.gid', ondelete='SET NULL'), index=True)
-    country = db.relationship('Country', foreign_keys=[country_id], backref=db.backref('stats', order_by='CountryStats.date.asc()'))
-
+    country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="SET NULL"), index=True)
+    country = db.relationship("Country", foreign_keys=[country_id], backref=db.backref("stats", order_by="CountryStats.date.asc()"))

app/model/device.py

@@ -2,17 +2,17 @@ import datetime
 
 from sqlalchemy.ext.hybrid import hybrid_property
 
-from ogn_python import db
+from app import db
 from .device_info import DeviceInfo
 
 
 class Device(db.Model):
-    __tablename__ = 'devices'
+    __tablename__ = "devices"
 
     id = db.Column(db.Integer, primary_key=True)
 
     name = db.Column(db.String, index=True)
-    #address = db.Column(db.String(6), index=True)
+    # address = db.Column(db.String(6), index=True)
     address = db.Column(db.String, index=True)
     firstseen = db.Column(db.DateTime, index=True)
     lastseen = db.Column(db.DateTime, index=True)
@@ -23,26 +23,16 @@ class Device(db.Model):
     real_address = db.Column(db.String(6))
 
     def __repr__(self):
-        return "<Device: %s,%s,%s,%s,%s,%s>" % (
-            self.address,
-            self.aircraft_type,
-            self.stealth,
-            self.software_version,
-            self.hardware_version,
-            self.real_address)
+        return "<Device: %s,%s,%s,%s,%s,%s>" % (self.address, self.aircraft_type, self.stealth, self.software_version, self.hardware_version, self.real_address)
 
     @hybrid_property
     def info(self):
-        query = db.session.query(DeviceInfo) \
-            .filter(DeviceInfo.address == self.address) \
-            .order_by(DeviceInfo.address_origin)
+        query = db.session.query(DeviceInfo).filter(DeviceInfo.address == self.address).order_by(DeviceInfo.address_origin)
 
         return query.first()
 
     def get_infos(self):
-        query = db.session.query(DeviceInfo) \
-            .filter(DeviceInfo.address == self.address) \
-            .order_by(DeviceInfo.address_origin)
+        query = db.session.query(DeviceInfo).filter(DeviceInfo.address == self.address).order_by(DeviceInfo.address_origin)
 
         return [info for info in query.all()]
 
@@ -62,7 +52,7 @@ class Device(db.Model):
     }
 
     def expiry_date(self):
-        if self.name.startswith('FLR'):
+        if self.name.startswith("FLR"):
             if self.software_version in self.EXPIRY_DATES:
                 return self.EXPIRY_DATES[self.software_version]
             else:

app/model/device_info.py

@@ -1,12 +1,12 @@
-from ogn_python import db
+from app import db
 
 
 class DeviceInfo(db.Model):
-    __tablename__ = 'device_infos'
+    __tablename__ = "device_infos"
 
     id = db.Column(db.Integer, primary_key=True)
     address_type = None
-    #address = db.Column(db.String(6), index=True)
+    # address = db.Column(db.String(6), index=True)
     address = db.Column(db.String, index=True)
     aircraft = db.Column(db.String)
     registration = db.Column(db.String(7))
@@ -27,4 +27,5 @@ class DeviceInfo(db.Model):
             self.tracked,
             self.identified,
             self.aircraft_type,
-            self.address_origin)
+            self.address_origin,
+        )

app/model/device_info_origin.py

@@ -8,15 +8,15 @@ class DeviceInfoOrigin:
         if origin in [0, 1, 2, 3]:
             self.origin = origin
         else:
-            raise ValueError('no address origin with id {} known'.format(origin))
+            raise ValueError("no address origin with id {} known".format(origin))
 
     def name(self):
         if self.origin == self.unknown:
-            return 'unknown'
+            return "unknown"
         elif self.origin == self.ogn_ddb:
-            return 'OGN-DDB'
+            return "OGN-DDB"
         elif self.origin == self.flarmnet:
-            return 'FlarmNet'
+            return "FlarmNet"
         elif self.origin == self.user_defined:
-            return 'user-defined'
-        return ''
+            return "user-defined"
+        return ""

app/model/device_stats.py

@@ -1,4 +1,4 @@
-from ogn_python import db
+from app import db
 
 
 class DeviceStats(db.Model):
@@ -40,15 +40,11 @@ class DeviceStats(db.Model):
     quality_ranking_country = db.Column(db.Integer)
 
     # Relations
-    device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), index=True)
-    device = db.relationship('Device', foreign_keys=[device_id], backref=db.backref('stats', order_by='DeviceStats.date.asc()'))
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="SET NULL"), index=True)
+    device = db.relationship("Device", foreign_keys=[device_id], backref=db.backref("stats", order_by="DeviceStats.date.asc()"))
 
     def __repr__(self):
-        return "<DeviceStats: %s,%s,%s,%s>" % (
-            self.date,
-            self.receiver_count,
-            self.aircraft_beacon_count,
-            self.max_altitude)
+        return "<DeviceStats: %s,%s,%s,%s>" % (self.date, self.receiver_count, self.aircraft_beacon_count, self.max_altitude)
 
 
-db.Index('ix_device_stats_date_device_id', DeviceStats.date, DeviceStats.device_id)
+db.Index("ix_device_stats_date_device_id", DeviceStats.date, DeviceStats.device_id)

app/model/flights2d.py

@@ -0,0 +1,24 @@
+from geoalchemy2.types import Geometry
+
+from app import db
+
+
+class Flight2D(db.Model):
+    __tablename__ = "flights2d"
+
+    date = db.Column(db.Date, primary_key=True)
+    flight_type = db.Column(db.SmallInteger, primary_key=True)
+
+    path_wkt = db.Column("path", Geometry("MULTILINESTRING", srid=4326))
+    path_simple_wkt = db.Column("path_simple", Geometry("MULTILINESTRING", srid=4326))  # this is the path simplified with ST_Simplify(path, 0.0001)
+
+    # Relations
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="SET NULL"), primary_key=True)
+    device = db.relationship("Device", foreign_keys=[device_id], backref="flights2d")
+
+    def __repr__(self):
+        return "<Flight %s: %s,%s>" % (self.date, self.path_wkt, self.path_simple_wkt)
+
+
+db.Index("ix_flights2d_date_device_id", Flight2D.date, Flight2D.device_id)
+# db.Index('ix_flights2d_date_path', Flight2D.date, Flight2D.path_wkt) --> CREATE INDEX ix_flights2d_date_path ON flights2d USING GIST("date", path)

app/model/geo.py

@@ -6,10 +6,10 @@ class Location:
         self.latitude = lat
 
     def to_wkt(self):
-        return 'SRID=4326;POINT({0} {1})'.format(self.longitude, self.latitude)
+        return "SRID=4326;POINT({0} {1})".format(self.longitude, self.latitude)
 
     def __str__(self):
-        return '{0: 7.4f}, {1:8.4f}'.format(self.latitude, self.longitude)
+        return "{0: 7.4f}, {1:8.4f}".format(self.latitude, self.longitude)
 
     def as_dict(self):
-        return {'latitude': round(self.latitude, 8), 'longitude': round(self.longitude, 8)}
+        return {"latitude": round(self.latitude, 8), "longitude": round(self.longitude, 8)}

app/model/logbook.py

@@ -1,10 +1,10 @@
 from sqlalchemy.ext.hybrid import hybrid_property
 
-from ogn_python import db
+from app import db
 
 
 class Logbook(db.Model):
-    __tablename__ = 'logbook'
+    __tablename__ = "logbook"
 
     id = db.Column(db.Integer, primary_key=True)
 
@@ -16,14 +16,14 @@ class Logbook(db.Model):
     max_altitude = db.Column(db.Float(precision=2))
 
     # Relations
-    takeoff_airport_id = db.Column(db.Integer, db.ForeignKey('airports.id', ondelete='CASCADE'), index=True)
-    takeoff_airport = db.relationship('Airport', foreign_keys=[takeoff_airport_id])
+    takeoff_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True)
+    takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id])
 
-    landing_airport_id = db.Column(db.Integer, db.ForeignKey('airports.id', ondelete='CASCADE'), index=True)
-    landing_airport = db.relationship('Airport', foreign_keys=[landing_airport_id])
+    landing_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True)
+    landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id])
 
-    device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='CASCADE'), index=True)
-    device = db.relationship('Device', foreign_keys=[device_id], backref=db.backref('logbook', order_by='Logbook.reftime'))
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="CASCADE"), index=True)
+    device = db.relationship("Device", foreign_keys=[device_id], backref=db.backref("logbook", order_by="Logbook.reftime"))
 
     @hybrid_property
     def duration(self):

app/model/receiver.py

@@ -3,7 +3,7 @@ from geoalchemy2.types import Geometry
 
 from .geo import Location
 
-from ogn_python import db
+from app import db
 
 
 class Receiver(db.Model):
@@ -11,7 +11,7 @@ class Receiver(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
 
-    location_wkt = db.Column('location', Geometry('POINT', srid=4326))
+    location_wkt = db.Column("location", Geometry("POINT", srid=4326))
     altitude = db.Column(db.Float(precision=2))
 
     name = db.Column(db.String(9), index=True)
@@ -21,8 +21,8 @@ class Receiver(db.Model):
     platform = db.Column(db.String)
 
     # Relations
-    country_id = db.Column(db.Integer, db.ForeignKey('countries.gid', ondelete='SET NULL'), index=True)
-    country = db.relationship('Country', foreign_keys=[country_id], backref=db.backref('receivers', order_by='Receiver.name.asc()'))
+    country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="SET NULL"), index=True)
+    country = db.relationship("Country", foreign_keys=[country_id], backref=db.backref("receivers", order_by="Receiver.name.asc()"))
 
     @property
     def location(self):

app/model/receiver_beacon.py

@@ -1,7 +1,7 @@
 from sqlalchemy.sql import func
 from .beacon import Beacon
 
-from ogn_python import db
+from app import db
 
 
 class ReceiverBeacon(Beacon):
@@ -35,11 +35,11 @@ class ReceiverBeacon(Beacon):
     user_comment = None
 
     # Relations
-    receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'))
-    receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_beacons')
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="SET NULL"))
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref="receiver_beacons")
 
     # Multi-column indices
-    db.Index('ix_receiver_beacons_receiver_id_name', 'receiver_id', 'name')
+    db.Index("ix_receiver_beacons_receiver_id_name", "receiver_id", "name")
 
     def __repr__(self):
         return "<ReceiverBeacon %s: %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
@@ -60,38 +60,39 @@ class ReceiverBeacon(Beacon):
             self.senders_messages,
             self.good_senders_signal,
             self.good_senders,
-            self.good_and_bad_senders)
+            self.good_and_bad_senders,
+        )
 
     @classmethod
     def get_columns(self):
-        return ['location',
-                'altitude',
-                'name',
-                'dstcall',
-                'receiver_name',
-                'timestamp',
-
-                # 'raw_message',
-                # 'reference_timestamp',
-
-                'version',
-                'platform',
-                'cpu_load',
-                'free_ram',
-                'total_ram',
-                'ntp_error',
-                'rt_crystal_correction',
-                'voltage',
-                'amperage',
-                'cpu_temp',
-                'senders_visible',
-                'senders_total',
-                'rec_input_noise',
-                'senders_signal',
-                'senders_messages',
-                'good_senders_signal',
-                'good_senders',
-                'good_and_bad_senders']
+        return [
+            "location",
+            "altitude",
+            "name",
+            "dstcall",
+            "receiver_name",
+            "timestamp",
+            # 'raw_message',
+            # 'reference_timestamp',
+            "version",
+            "platform",
+            "cpu_load",
+            "free_ram",
+            "total_ram",
+            "ntp_error",
+            "rt_crystal_correction",
+            "voltage",
+            "amperage",
+            "cpu_temp",
+            "senders_visible",
+            "senders_total",
+            "rec_input_noise",
+            "senders_signal",
+            "senders_messages",
+            "good_senders_signal",
+            "good_senders",
+            "good_and_bad_senders",
+        ]
 
     def get_values(self):
         return [
@@ -101,10 +102,8 @@ class ReceiverBeacon(Beacon):
             self.dstcall,
             self.receiver_name,
             self.timestamp,
-
             # self.raw_message,
             # self.reference_timestamp,
-
             self.version,
             self.platform,
             self.cpu_load,
@@ -122,7 +121,8 @@ class ReceiverBeacon(Beacon):
             int(self.senders_messages) if self.senders_messages else None,
             self.good_senders_signal,
             int(self.good_senders) if self.good_senders else None,
-            int(self.good_and_bad_senders) if self.good_and_bad_senders else None]
+            int(self.good_and_bad_senders) if self.good_and_bad_senders else None,
+        ]
 
 
-db.Index('ix_receiver_beacons_date_receiver_id', func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id)
+db.Index("ix_receiver_beacons_date_receiver_id", func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id)

app/model/receiver_coverage.py

@@ -1,4 +1,4 @@
-from ogn_python import db
+from app import db
 
 
 class ReceiverCoverage(db.Model):
@@ -15,9 +15,9 @@ class ReceiverCoverage(db.Model):
     device_count = db.Column(db.SmallInteger)
 
     # Relations
-    receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'), primary_key=True)
-    receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref=db.backref('receiver_coverages', order_by='ReceiverCoverage.date.asc()'))
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="SET NULL"), primary_key=True)
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("receiver_coverages", order_by="ReceiverCoverage.date.asc()"))
 
 
-db.Index('ix_receiver_coverages_date_receiver_id', ReceiverCoverage.date, ReceiverCoverage.receiver_id)
-db.Index('ix_receiver_coverages_receiver_id_date', ReceiverCoverage.receiver_id, ReceiverCoverage.date)
+db.Index("ix_receiver_coverages_date_receiver_id", ReceiverCoverage.date, ReceiverCoverage.receiver_id)
+db.Index("ix_receiver_coverages_receiver_id_date", ReceiverCoverage.receiver_id, ReceiverCoverage.date)

app/model/receiver_stats.py

@@ -1,6 +1,6 @@
 from geoalchemy2.types import Geometry
 
-from ogn_python import db
+from app import db
 
 
 class ReceiverStats(db.Model):
@@ -13,7 +13,7 @@ class ReceiverStats(db.Model):
     # Static data
     firstseen = db.Column(db.DateTime, index=True)
     lastseen = db.Column(db.DateTime, index=True)
-    location_wkt = db.Column('location', Geometry('POINT', srid=4326))
+    location_wkt = db.Column("location", Geometry("POINT", srid=4326))
     altitude = db.Column(db.Float(precision=2))
     version = db.Column(db.String)
     platform = db.Column(db.String)
@@ -34,8 +34,8 @@ class ReceiverStats(db.Model):
     quality_ranking = db.Column(db.Integer)
 
     # Relations
-    receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'), index=True)
-    receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref=db.backref('stats', order_by='ReceiverStats.date.asc()'))
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="SET NULL"), index=True)
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("stats", order_by="ReceiverStats.date.asc()"))
 
 
-db.Index('ix_receiver_stats_date_receiver_id', ReceiverStats.date, ReceiverStats.receiver_id)
+db.Index("ix_receiver_stats_date_receiver_id", ReceiverStats.date, ReceiverStats.receiver_id)

app/model/relation_stats.py

@@ -0,0 +1,26 @@
+from app import db
+
+
+class RelationStats(db.Model):
+    __tablename__ = "relation_stats"
+
+    id = db.Column(db.Integer, primary_key=True)
+
+    date = db.Column(db.Date)
+
+    # Statistic data
+    quality = db.Column(db.Float(precision=2))
+    beacon_count = db.Column(db.Integer)
+
+    # Relations
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="SET NULL"), index=True)
+    device = db.relationship("Device", foreign_keys=[device_id], backref="relation_stats")
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="SET NULL"), index=True)
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref="relation_stats")
+
+    def __repr__(self):
+        return "<RelationStats: %s,%s,%s>" % (self.date, self.quality, self.beacon_count)
+
+
+db.Index("ix_relation_stats_date_device_id", RelationStats.date, RelationStats.device_id, RelationStats.receiver_id)
+db.Index("ix_relation_stats_date_receiver_id", RelationStats.date, RelationStats.receiver_id, RelationStats.device_id)

app/model/takeoff_landing.py

@@ -0,0 +1,16 @@
+from app import db
+
+
+class TakeoffLanding(db.Model):
+    __tablename__ = "takeoff_landings"
+
+    device_id = db.Column(db.Integer, db.ForeignKey("devices.id", ondelete="SET NULL"), primary_key=True)
+    airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="SET NULL"), primary_key=True)
+    timestamp = db.Column(db.DateTime, primary_key=True)
+
+    is_takeoff = db.Column(db.Boolean)
+    track = db.Column(db.SmallInteger)
+
+    # Relations
+    airport = db.relationship("Airport", foreign_keys=[airport_id], backref="takeoff_landings")
+    device = db.relationship("Device", foreign_keys=[device_id], backref="takeoff_landings", order_by="TakeoffLanding.timestamp")

app/routes.py

@@ -0,0 +1,197 @@
+import datetime
+
+from flask import request, render_template, send_file
+
+from app import app
+from app import db
+from app import cache
+
+from app.model import *
+
+
+@cache.cached(key_prefix="countries_in_receivers")
+def get_countries_in_receivers():
+    query = db.session.query(Country.iso2).filter(Country.gid == Receiver.country_id).order_by(Country.iso2).distinct(Country.iso2)
+
+    return [{"iso2": country[0]} for country in query.all()]
+
+
+@cache.cached(key_prefix="countries_in_logbook")
+def get_countries_in_logbook():
+    query = db.session.query(Country.iso2).filter(Country.iso2 == Airport.country_code).filter(Logbook.takeoff_airport_id == Airport.id).order_by(Country.iso2).distinct(Country.iso2)
+
+    return [{"iso2": country[0]} for country in query.all()]
+
+
+@cache.memoize()
+def get_airports_in_country(sel_country):
+    query = db.session.query(Airport.id, Airport.name).filter(Airport.country_code == sel_country).filter(Logbook.takeoff_airport_id == Airport.id).order_by(Airport.name).distinct(Airport.name)
+
+    return [{"id": airport[0], "name": airport[1]} for airport in query.all()]
+
+
+@cache.memoize()
+def get_dates_for_airport(sel_airport):
+    query = (
+        db.session.query(db.func.date(Logbook.reftime), db.func.count(Logbook.id).label("logbook_count"))
+        .filter(Airport.id == sel_airport)
+        .filter(db.or_(Airport.id == Logbook.takeoff_airport_id, Airport.id == Logbook.landing_airport_id))
+        .group_by(db.func.date(Logbook.reftime))
+        .order_by(db.func.date(Logbook.reftime).desc())
+    )
+
+    return [{"date": date, "logbook_count": logbook_count} for (date, logbook_count) in query.all()]
+
+
+@app.route("/")
+@app.route("/index.html")
+def index():
+    return render_template("base.html")
+
+
+@app.route("/devices.html", methods=["GET", "POST"])
+def devices():
+    devices = db.session.query(Device).order_by(Device.address).limit(100)
+    return render_template("devices.html", devices=devices)
+
+
+@app.route("/device_detail.html", methods=["GET", "POST"])
+def device_detail():
+    device_id = request.args.get("id")
+    device = db.session.query(Device).filter(Device.id == device_id).one()
+
+    return render_template("device_detail.html", title="Device", device=device)
+
+
+@app.route("/receivers.html")
+def receivers():
+    sel_country = request.args.get("country")
+
+    countries = get_countries_in_receivers()
+
+    # Get receiver selection list
+    if sel_country:
+        receivers = db.session.query(Receiver).filter(db.and_(Receiver.country_id == Country.gid, Country.iso2 == sel_country)).order_by(Receiver.name)
+    else:
+        receivers = db.session.query(Receiver).order_by(Receiver.name)
+
+    return render_template("receivers.html", title="Receivers", sel_country=sel_country, countries=countries, receivers=receivers)
+
+
+@app.route("/receiver_detail.html")
+def receiver_detail():
+    sel_receiver_id = request.args.get("receiver_id")
+
+    receiver = db.session.query(Receiver).filter(Receiver.id == sel_receiver_id).one()
+
+    airport = (
+        db.session.query(Airport)
+        .filter(
+            db.and_(
+                Receiver.id == sel_receiver_id,
+                db.func.st_contains(db.func.st_buffer(Receiver.location_wkt, 0.5), Airport.location_wkt),
+                db.func.st_distance_sphere(Airport.location_wkt, Receiver.location_wkt) < 1000,
+            )
+        )
+        .filter(Airport.style.in_((2, 4, 5)))
+    )
+    return render_template("receiver_detail.html", title="Receiver Detail", receiver=receiver, airport=airport.first())
+
+
+@app.route("/airports.html", methods=["GET", "POST"])
+def airports():
+    sel_country = request.args.get("country")
+
+    countries = get_countries_in_logbook()
+
+    if sel_country:
+        airports = get_airports_in_country(sel_country)
+    else:
+        airports = []
+
+    page = request.args.get("page", 1, type=int)
+
+    return render_template("airports.html", sel_country=sel_country, countries=countries, airports=airports)
+
+
+@app.route("/airport_detail.html")
+def airport_detail():
+    sel_airport = request.args.get("airport")
+
+    airport = db.session.query(Airport).filter(Airport.id == sel_airport)
+
+    devices = db.session.query(Device).join(Logbook).filter(Logbook.takeoff_airport_id == sel_airport).order_by(Device.address)
+
+    return render_template("airport_detail.html", title="Airport Detail", airport=airport.one(), devices=devices)
+
+
+@app.route("/logbook.html", methods=["GET", "POST"])
+def logbook():
+    sel_country = request.args.get("country")
+    sel_airport = request.args.get("airport")
+    sel_date = request.args.get("date")
+
+    sel_device_id = request.args.get("device_id")
+
+    countries = get_countries_in_logbook()
+
+    if sel_country:
+        airports = get_airports_in_country(sel_country)
+    else:
+        airports = []
+
+    if sel_airport:
+        sel_airport = int(sel_airport)
+        if sel_airport not in [airport["id"] for airport in airports]:
+            sel_airport = None
+            sel_date = None
+        dates = get_dates_for_airport(sel_airport)
+    else:
+        dates = []
+
+    if sel_date:
+        sel_date = datetime.datetime.strptime(sel_date, "%Y-%m-%d").date()
+        if sel_date not in [entry["date"] for entry in dates]:
+            sel_date = dates[0]["date"]
+    elif len(dates) > 0:
+        sel_date = dates[0]["date"]
+
+    # Get Logbook
+    filters = []
+    if sel_airport:
+        filters.append(db.or_(Logbook.takeoff_airport_id == sel_airport, Logbook.landing_airport_id == sel_airport))
+
+    if sel_date:
+        filters.append(db.func.date(Logbook.reftime) == sel_date)
+
+    if sel_device_id:
+        filters.append(Logbook.device_id == sel_device_id)
+
+    if len(filters) > 0:
+        logbook = db.session.query(Logbook).filter(*filters).order_by(Logbook.reftime)
+    else:
+        logbook = None
+
+    return render_template("logbook.html", title="Logbook", sel_country=sel_country, countries=countries, sel_airport=sel_airport, airports=airports, sel_date=sel_date, dates=dates, logbook=logbook)
+
+
+@app.route("/download.html")
+def download_flight():
+    from io import StringIO
+
+    buffer = StringIO()
+    buffer.write("Moin moin\nAlter Verwalter")
+    buffer.seek(0)
+
+    return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain")
+
+
+@app.route("/statistics.html")
+def statistics():
+
+    today = datetime.date.today()
+    today = datetime.date(2018, 7, 31)
+
+    receiverstats = db.session.query(ReceiverStats).filter(ReceiverStats.date == today)
+
+    return render_template("statistics.html", title="Receiver Statistics", receiverstats=receiverstats)