diff --git a/README.md b/README.md index bbb692a..7c38459 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,13 @@ For best performance you should use [TimescaleDB](https://www.timescale.com), wh 5. Create database ``` - ./manage.py db.init + ./flask database init ``` 6. Optional: Prepare tables for TimescaleDB ``` - ./manage.py db.init_timescaledb + ./flask database init_timescaledb ``` 7. Optional: Import world border dataset (needed if you want to know the country a receiver belongs to, etc.) @@ -85,7 +85,7 @@ The following scripts run in the foreground and should be deamonized - Start the aprs client ``` - ./manage.py gateway.run + ./flask gateway run ``` - Start a task server (make sure redis is up and running) @@ -107,66 +107,44 @@ and set the environment variable `OGN_CONFIG_MODULE` accordingly. ``` touch myconfig.py export OGN_CONFIG_MODULE="myconfig" -./manage.py gateway.run +./flask gateway run ``` -### manage.py - CLI options +### Flask - Command Line Interface ``` -usage: manage [.] [] +Usage: flask [OPTIONS] COMMAND [ARGS]... -positional arguments: - command the command to run + A general utility script for Flask applications. -optional arguments: - -h, --help show this help message and exit + Provides commands from Flask, extensions, and the application. Loads the + application defined in the FLASK_APP environment variable, or from a + wsgi.py file. Setting the FLASK_ENV environment variable to 'development' + will enable debug mode. -available commands: - - [bulkimport] - create_flights2d Create complete flight traces from logfile tables. - create_gaps2d Create 'gaps' from logfile tables. - file_export Export separate logfile tables to csv files. They can be used for fast bulk import with sql COPY command. - file_import Import APRS logfiles into separate logfile tables. - transfer Transfer beacons from separate logfile tables to beacon table. - update Update beacons (add foreign keys, compute distance, bearing, ags, etc.) in separate logfile tables. - - [db] - drop Drop all tables. - import_airports Import airports from a ".cup" file - import_ddb Import registered devices from the DDB. - import_file Import registered devices from a local file. - import_flarmnet Import registered devices from a local file. - init Initialize the database. - init_timescaledb Initialize TimescaleDB features. - update_country_codes Update country codes of all receivers. - upgrade Upgrade database to the latest version. - - [flights] - flights2d Compute flights. - - [gateway] - run Run the aprs client. - - [export] - cup Export receiver waypoints as '.cup'. - igc Export igc file for
at . - - [logbook] - compute_logbook Compute logbook. - compute_takeoff_landingCompute takeoffs and landings. - show Show a logbook for . - - [stats] - create Create DeviceStats, ReceiverStats and RelationStats. - create_ognrange Create stats for Melissa's ognrange. - update_devices Update devices with data from stats. - update_receivers Update receivers with data from stats. + $ export FLASK_APP=app.py + $ export FLASK_ENV=development + $ flask run + +Options: + --version Show the flask version + --help Show this message and exit. + +Commands: + database Database creation and handling. + db Perform database migrations. + export Export data in several file formats. + flights Create 2D flight paths from data. + gateway Connection to APRS servers. + logbook Handling of logbook data. + routes Show the routes for the app. + run Runs a development server. + shell Runs a shell in the app context. + stats Handling of statistical data. ``` -Only the command `logbook.compute` requires a running task server (celery) at the moment. +Most commands are command groups, so if you execute this command you will get further (sub)commands. - -### Available tasks +### Available tasks (deprecated - needs rework) - `ogn.collect.database.import_ddb` - Import registered devices from the DDB. - `ogn.collect.database.import_file` - Import registered devices from a local file. diff --git a/manage.py b/manage.py deleted file mode 100755 index 6dbfac2..0000000 --- a/manage.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -from manager import Manager -from ogn.commands import manager as command_manager -from ogn.gateway.manage import manager as gateway_manager - -manager = Manager() -manager.merge(command_manager) -manager.merge(gateway_manager, namespace='gateway') - - -if __name__ == '__main__': - manager.main() diff --git a/ogn/collect/database.py b/ogn/collect/database.py deleted file mode 100644 index 8e239be..0000000 --- a/ogn/collect/database.py +++ /dev/null @@ -1,164 +0,0 @@ -from celery.utils.log import get_task_logger - -from sqlalchemy import distinct -from sqlalchemy.sql import null, and_, func, not_, case -from sqlalchemy.dialects import postgresql -from sqlalchemy.dialects.postgresql import insert - -from ogn.collect.celery import app -from ogn.model import Country, DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver -from ogn.utils import get_ddb, get_flarmnet - - -logger = get_task_logger(__name__) - - -def compile_query(query): - """Via http://nicolascadou.com/blog/2014/01/printing-actual-sqlalchemy-queries""" - compiler = query.compile if not hasattr(query, 'statement') else query.statement.compile - return compiler(dialect=postgresql.dialect()) - - -def upsert(session, model, rows, update_cols): - """Insert rows in model. On conflicting update columns if new value IS NOT NULL.""" - - table = model.__table__ - - stmt = insert(table).values(rows) - - on_conflict_stmt = stmt.on_conflict_do_update( - index_elements=table.primary_key.columns, - set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols}, - ) - - # print(compile_query(on_conflict_stmt)) - session.execute(on_conflict_stmt) - - -def update_device_infos(session, address_origin, path=None): - if address_origin == DeviceInfoOrigin.flarmnet: - device_infos = get_flarmnet(fln_file=path) - else: - device_infos = get_ddb(csv_file=path) - - session.query(DeviceInfo) \ - .filter(DeviceInfo.address_origin == address_origin) \ - .delete(synchronize_session='fetch') - session.commit() - - for device_info in device_infos: - device_info.address_origin = address_origin - - session.bulk_save_objects(device_infos) - session.commit() - - return len(device_infos) - - -@app.task -def import_ddb(session=None): - """Import registered devices from the DDB.""" - - if session is None: - session = app.session - - logger.info("Import registered devices fom the DDB...") - counter = update_device_infos(session, DeviceInfoOrigin.ogn_ddb) - logger.info("Imported {} devices.".format(counter)) - - return "Imported {} devices.".format(counter) - - -@app.task -def add_missing_devices(session=None): - """Add/update entries in devices table and update foreign keys in aircraft beacons.""" - - if session is None: - session = app.session - - # Create missing Device from AircraftBeacon - available_devices = session.query(Device.address) \ - .subquery() - - missing_devices_query = session.query(distinct(AircraftBeacon.address)) \ - .filter(and_(AircraftBeacon.device_id == null(), not_(AircraftBeacon.address.like('00%')), AircraftBeacon.error_count == 0)) \ - .filter(~AircraftBeacon.address.in_(available_devices)) - - ins = insert(Device).from_select([Device.address], missing_devices_query) - res = session.execute(ins) - insert_count = res.rowcount - session.commit() - - # Update relations to aircraft beacons - upd = session.query(AircraftBeacon) \ - .filter(AircraftBeacon.device_id == null()) \ - .filter(AircraftBeacon.address == Device.address) \ - .update({ - AircraftBeacon.device_id: Device.id}, - synchronize_session='fetch') - - session.commit() - logger.info("Devices: {} inserted, {} updated".format(insert_count, add_missing_receivers)) - logger.info("Updated {} AircraftBeacons".format(upd)) - - return "{} Devices inserted, {} Devices updated, {} AircraftBeacons updated" \ - .format(insert_count, add_missing_receivers, upd) - - -@app.task -def add_missing_receivers(session=None): - """Add/add_missing_receivers entries in receiver table and update receivers foreign keys and distance in aircraft beacons and update foreign keys in receiver beacons.""" - - if session is None: - session = app.session - - # Create missing Receiver from ReceiverBeacon - available_receivers = session.query(Receiver.name) \ - .subquery() - - missing_receiver_query = session.query(distinct(ReceiverBeacon.name)) \ - .filter(ReceiverBeacon.receiver_id == null()) \ - .filter(~ReceiverBeacon.name.in_(available_receivers)) - - ins = insert(Receiver).from_select([Receiver.name], missing_receiver_query) - res = session.execute(ins) - insert_count = res.rowcount - - # Update relations to aircraft beacons - update_aircraft_beacons = session.query(AircraftBeacon) \ - .filter(and_(AircraftBeacon.receiver_id == null(), AircraftBeacon.receiver_name == Receiver.name)) \ - .update({AircraftBeacon.receiver_id: Receiver.id, - AircraftBeacon.distance: func.ST_Distance_Sphere(AircraftBeacon.location_wkt, Receiver.location_wkt)}, - synchronize_session='fetch') - - # Update relations to receiver beacons - update_receiver_beacons = session.query(ReceiverBeacon) \ - .filter(and_(ReceiverBeacon.receiver_id == null(), ReceiverBeacon.name == Receiver.name)) \ - .update({ReceiverBeacon.receiver_id: Receiver.id}, - synchronize_session='fetch') - - session.commit() - - logger.info("Receivers: {} inserted, {} updated.".format(insert_count, add_missing_receivers)) - logger.info("Updated relations: {} aircraft beacons, {} receiver beacons".format(update_aircraft_beacons, update_receiver_beacons)) - - return "{} Receivers inserted, {} Receivers updated, {} AircraftBeacons updated, {} ReceiverBeacons updated" \ - .format(insert_count, add_missing_receivers, update_aircraft_beacons, update_receiver_beacons) - - -@app.task -def update_country_code(session=None): - """Update country code in receivers table if None.""" - - if session is None: - session = app.session - - update_receivers = session.query(Receiver) \ - .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom))) \ - .update({Receiver.country_id: Country.gid}, - synchronize_session='fetch') - - session.commit() - logger.info("Updated {} AircraftBeacons".format(update_receivers)) - - return "Updated country for {} Receivers".format(update_receivers) diff --git a/ogn/commands/__init__.py b/ogn/commands/__init__.py deleted file mode 100644 index 2ce4e2f..0000000 --- a/ogn/commands/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from .database import manager as database_manager -from .bulkimport import manager as bulkimport_manager -from .export import manager as export_manager -from .logbook import manager as logbook_manager -from .stats import manager as stats_manager -from .flights import manager as flights_manager - -from manager import Manager - -manager = Manager() - -manager.merge(database_manager, namespace='db') -manager.merge(bulkimport_manager, namespace='bulkimport') -manager.merge(export_manager, namespace='export') -manager.merge(logbook_manager, namespace='logbook') -manager.merge(stats_manager, namespace='stats') -manager.merge(flights_manager, namespace='flights') diff --git a/ogn/commands/bulkimport.py b/ogn/commands/bulkimport.py deleted file mode 100644 index a9d56e9..0000000 --- a/ogn/commands/bulkimport.py +++ /dev/null @@ -1,585 +0,0 @@ -from manager import Manager - -import psycopg2 -from tqdm import tqdm -from io import StringIO - -from ogn.model import AircraftBeacon, ReceiverBeacon -from ogn.utils import open_file -from ogn.commands.database import get_database_days - -manager = Manager() - - -class LogfileDbSaver(): - def __init__(self): - """Establish the database connection.""" - try: - self.conn = psycopg2.connect(database="ogn", user="postgres", password="postgres", host="localhost", port="5432") - except Exception as e: - raise Exception("I am unable to connect to the database") - self.cur = self.conn.cursor() - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - """Closes the database connection.""" - - self.cur.close() - self.conn.close() - - def set_datestr(self, datestr): - """Sets the datestr of the current tables.""" - - self.prefix = datestr.replace('-', '_') - self.aircraft_table = 'aircraft_beacons_{}'.format(self.prefix) - self.receiver_table = 'receiver_beacons_{}'.format(self.prefix) - self.aircraft_buffer = StringIO() - self.receiver_buffer = StringIO() - - def get_datestrs(self, no_index_only=False): - """Get the date strings from imported log files.""" - - index_clause = " AND hasindexes = FALSE" if no_index_only else "" - - self.cur.execute((""" - SELECT DISTINCT(RIGHT(tablename, 10)) - FROM pg_catalog.pg_tables - WHERE schemaname = 'public' AND tablename LIKE 'aircraft_beacons_%'{} - ORDER BY RIGHT(tablename, 10); - """.format(index_clause))) - - return [datestr[0].replace('_', '-') for datestr in self.cur.fetchall()] - - def create_tables(self): - """Create date dependent tables for log file import.""" - - try: - self.cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - self.cur.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;') - self.cur.execute('DROP TABLE IF EXISTS "{0}"; CREATE TABLE {0} AS TABLE aircraft_beacons WITH NO DATA;'.format(self.aircraft_table)) - self.cur.execute('DROP TABLE IF EXISTS "{0}"; CREATE TABLE {0} AS TABLE receiver_beacons WITH NO DATA;'.format(self.receiver_table)) - self.conn.commit() - except Exception as e: - raise Exception("I can't create the tables") - - def add(self, beacon): - """Adds the values of the beacon to the buffer.""" - - value_string = ','.join([str(value) for value in beacon.get_values()]) + '\n' - if isinstance(beacon, AircraftBeacon): - self.aircraft_buffer.write(value_string) - elif isinstance(beacon, ReceiverBeacon): - self.receiver_buffer.write(value_string) - - def flush(self): - """Writes the buffer into the tables and reset the buffer.""" - - self.aircraft_buffer.seek(0) - self.receiver_buffer.seek(0) - self.cur.copy_from(self.aircraft_buffer, self.aircraft_table, sep=',', null='None', columns=AircraftBeacon.get_columns()) - self.cur.copy_from(self.receiver_buffer, self.receiver_table, sep=',', null='None', columns=ReceiverBeacon.get_columns()) - self.conn.commit() - self.aircraft_buffer = StringIO() - self.receiver_buffer = StringIO() - - def export_to_path(self, path): - import os, gzip - aircraft_beacons_file = os.path.join(path, self.aircraft_table + '.csv.gz') - with gzip.open(aircraft_beacons_file, 'wt', encoding='utf-8') as gzip_file: - self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_aircraft_beacons_subquery()), gzip_file) - receiver_beacons_file = os.path.join(path, self.receiver_table + '.csv.gz') - with gzip.open(receiver_beacons_file, 'wt') as gzip_file: - self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_receiver_beacons_subquery()), gzip_file) - - def create_indices(self): - """Creates indices for aircraft- and receiver-beacons.""" - - self.cur.execute(""" - CREATE INDEX IF NOT EXISTS ix_{0}_timestamp_name_receiver_name ON "{0}" (timestamp, name, receiver_name); - CREATE INDEX IF NOT EXISTS ix_{0}_device_id_timestamp_error_count ON "{0}" (device_id, timestamp, error_count); - CREATE INDEX IF NOT EXISTS ix_{1}_timestamp_name_receiver_name ON "{1}" (timestamp, name, receiver_name); - """.format(self.aircraft_table, self.receiver_table)) - self.conn.commit() - - def add_missing_devices(self): - """Add missing devices.""" - - self.cur.execute(""" - INSERT INTO devices(address) - SELECT DISTINCT(ab.address) - FROM "{}" AS ab - WHERE NOT EXISTS (SELECT 1 FROM devices AS d WHERE d.address = ab.address) - ORDER BY ab.address; - """.format(self.aircraft_table)) - self.conn.commit() - - def add_missing_receivers(self): - """Add missing receivers.""" - - self.cur.execute(""" - INSERT INTO receivers(name) - SELECT DISTINCT(rb.name) - FROM "{0}" AS rb - WHERE NOT EXISTS (SELECT 1 FROM receivers AS r WHERE r.name = rb.name) - ORDER BY name; - """.format(self.receiver_table)) - self.conn.commit() - - def update_receiver_location(self): - """Updates the receiver location. We need this because we want the actual location for distance calculations.""" - - self.cur.execute(""" - UPDATE receivers AS r - SET location = sq.location, - altitude = sq.altitude - FROM - (SELECT DISTINCT ON (rb.receiver_id) rb.receiver_id, rb.location, rb.altitude - FROM "{1}" AS rb - WHERE rb.location IS NOT NULL - ORDER BY rb.receiver_id, rb.timestamp - ) AS sq - WHERE r.id = sq.receiver_id; - """.format(self.aircraft_table, self.receiver_table)) - self.conn.commit() - - def update_receiver_beacons(self): - """Updates the foreign keys. Due to performance reasons we use a new table instead of updating the old.""" - - self.cur.execute(""" - SELECT - - rb.location, rb.altitude, rb.name, rb.receiver_name, rb.dstcall, rb.timestamp, - - rb.version, rb.platform, rb.cpu_load, rb.free_ram, rb.total_ram, rb.ntp_error, rb.rt_crystal_correction, rb.voltage, rb.amperage real, - rb.cpu_temp, rb.senders_visible, rb.senders_total, rb.rec_input_noise, rb.senders_signal, rb.senders_messages, rb.good_senders_signal real, - rb.good_senders, rb.good_and_bad_senders, - - r.id AS receiver_id - INTO "{0}_temp" - FROM "{0}" AS rb, receivers AS r - WHERE rb.name = r.name; - - DROP TABLE IF EXISTS "{0}"; - ALTER TABLE "{0}_temp" RENAME TO "{0}"; - """.format(self.receiver_table)) - self.conn.commit() - - def update_aircraft_beacons(self): - """Updates the foreign keys and calculates distance/radial and quality and computes the altitude above ground level. - Elevation data has to be in the table 'elevation' with srid 4326. - Due to performance reasons we use a new table instead of updating the old.""" - - self.cur.execute(""" - SELECT - ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed, - - ab.address_type, ab.aircraft_type, ab.stealth, ab.address, ab.climb_rate, ab.turn_rate, ab.signal_quality, ab.error_count, - ab.frequency_offset, ab.gps_quality_horizontal, ab.gps_quality_vertical, ab.software_version, ab.hardware_version, ab.real_address, ab.signal_power, - - ab.location_mgrs, - ab.location_mgrs_short, - - d.id AS device_id, - r.id AS receiver_id, - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END AS distance, - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) ELSE NULL END AS radial, - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL - THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL) - ELSE NULL - END AS quality, - CAST(ab.altitude - ST_Value(e.rast, ab.location) AS REAL) AS agl - - INTO "{0}_temp" - FROM "{0}" AS ab, devices AS d, receivers AS r, elevation AS e - WHERE ab.address = d.address AND receiver_name = r.name AND ST_Intersects(e.rast, ab.location); - - DROP TABLE IF EXISTS "{0}"; - ALTER TABLE "{0}_temp" RENAME TO "{0}"; - """.format(self.aircraft_table)) - self.conn.commit() - - def get_merged_aircraft_beacons_subquery(self): - """Some beacons are split into position and status beacon. With this query we merge them into one beacon.""" - - return """ - SELECT - ST_AsEWKT(MAX(location)) AS location, - MAX(altitude) AS altitude, - name, - MAX(dstcall) AS dstcall, - MAX(relay) AS relay, - receiver_name, - timestamp, - MAX(track) AS track, - MAX(ground_speed) AS ground_speed, - - MAX(address_type) AS address_type, - MAX(aircraft_type) AS aircraft_type, - CAST(MAX(CAST(stealth AS int)) AS boolean) AS stealth, - MAX(address) AS address, - MAX(climb_rate) AS climb_rate, - MAX(turn_rate) AS turn_rate, - MAX(signal_quality) AS signal_quality, - MAX(error_count) AS error_count, - MAX(frequency_offset) AS frequency_offset, - MAX(gps_quality_horizontal) AS gps_quality_horizontal, - MAX(gps_quality_vertical) AS gps_quality_vertical, - MAX(software_version) AS software_version, - MAX(hardware_version) AS hardware_version, - MAX(real_address) AS real_address, - MAX(signal_power) AS signal_power, - - CAST(MAX(distance) AS REAL) AS distance, - CAST(MAX(radial) AS REAL) AS radial, - CAST(MAX(quality) AS REAL) AS quality, - CAST(MAX(agl) AS REAL) AS agl, - MAX(location_mgrs) AS location_mgrs, - MAX(location_mgrs_short) AS location_mgrs_short, - - MAX(receiver_id) AS receiver_id, - MAX(device_id) AS device_id - FROM "{0}" AS ab - GROUP BY timestamp, name, receiver_name - ORDER BY timestamp, name, receiver_name - """.format(self.aircraft_table) - - def get_merged_receiver_beacons_subquery(self): - """Some beacons are split into position and status beacon. With this query we merge them into one beacon.""" - - return """ - SELECT - ST_AsEWKT(MAX(location)) AS location, - MAX(altitude) AS altitude, - name, - receiver_name, - MAX(dstcall) AS dstcall, - timestamp, - - MAX(version) AS version, - MAX(platform) AS platform, - MAX(cpu_load) AS cpu_load, - MAX(free_ram) AS free_ram, - MAX(total_ram) AS total_ram, - MAX(ntp_error) AS ntp_error, - MAX(rt_crystal_correction) AS rt_crystal_correction, - MAX(voltage) AS voltage, - MAX(amperage) AS amperage, - MAX(cpu_temp) AS cpu_temp, - MAX(senders_visible) AS senders_visible, - MAX(senders_total) AS senders_total, - MAX(rec_input_noise) AS rec_input_noise, - MAX(senders_signal) AS senders_signal, - MAX(senders_messages) AS senders_messages, - MAX(good_senders_signal) AS good_senders_signal, - MAX(good_senders) AS good_senders, - MAX(good_and_bad_senders) AS good_and_bad_senders, - - MAX(receiver_id) AS receiver_id - FROM "{0}" AS rb - GROUP BY timestamp, name, receiver_name - ORDER BY timestamp, name, receiver_name - """.format(self.receiver_table) - - def is_transfered(self): - query = """ - SELECT - 1 - FROM ({} LIMIT 1) AS sq, aircraft_beacons AS ab - WHERE ab.timestamp = sq.timestamp AND ab.name = sq.name AND ab.receiver_name = sq.receiver_name; - """.format(self.get_merged_aircraft_beacons_subquery()) - - self.cur.execute(query) - return len(self.cur.fetchall()) == 1 - - def transfer_aircraft_beacons(self): - query = """ - INSERT INTO aircraft_beacons(location, altitude, name, dstcall, relay, receiver_name, timestamp, track, ground_speed, - address_type, aircraft_type, stealth, address, climb_rate, turn_rate, signal_quality, error_count, frequency_offset, gps_quality_horizontal, gps_quality_vertical, software_version, hardware_version, real_address, signal_power, - distance, radial, quality, agl, location_mgrs, location_mgrs_short, - receiver_id, device_id) - {} - ON CONFLICT DO NOTHING; - """.format(self.get_merged_aircraft_beacons_subquery()) - - self.cur.execute(query) - self.conn.commit() - - def transfer_receiver_beacons(self): - query = """ - INSERT INTO receiver_beacons(location, altitude, name, receiver_name, dstcall, timestamp, - - version, platform, cpu_load, free_ram, total_ram, ntp_error, rt_crystal_correction, voltage, - amperage, cpu_temp, senders_visible, senders_total, rec_input_noise, senders_signal, - senders_messages, good_senders_signal, good_senders, good_and_bad_senders, - - receiver_id) - {} - ON CONFLICT DO NOTHING; - """.format(self.get_merged_receiver_beacons_subquery()) - - self.cur.execute(query) - self.conn.commit() - - def create_flights2d(self): - query = """ - INSERT INTO flights2d - ( - date, - device_id, - path - ) - SELECT sq5.date, - sq5.device_id, - st_collect(sq5.linestring order BY sq5.part) multilinestring - FROM ( - SELECT sq4.timestamp::date AS date, - sq4.device_id, - sq4.part, - st_makeline(sq4.location ORDER BY sq4.timestamp) linestring - FROM ( - SELECT sq3.timestamp, - sq3.location, - sq3.device_id, - sum(sq3.ping) OVER (partition BY sq3.timestamp::date, sq3.device_id ORDER BY sq3.timestamp) part - FROM ( - SELECT sq2.t1 AS timestamp, - sq2.l1 AS location, - sq2.d1 device_id, - CASE - WHEN sq2.t1 - sq2.t2 < interval'100s' - AND st_distancesphere(sq2.l1, sq2.l2) < 1000 THEN 0 - ELSE 1 - END AS ping - FROM ( - SELECT sq.timestamp t1, - lag(sq.timestamp) OVER (partition BY sq.device_id ORDER BY sq.timestamp) t2, - sq.location l1, - lag(sq.location) OVER (partition BY sq.device_id ORDER BY sq.timestamp) l2, - sq.device_id d1, - lag(sq.device_id) OVER (partition BY sq.device_id ORDER BY sq.timestamp) d2 - FROM ( - SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location - FROM {} - WHERE device_id IS NOT NULL AND ground_speed > 250 AND agl < 100 - ORDER BY device_id, timestamp, error_count) sq) sq2 ) sq3 ) sq4 - GROUP BY sq4.timestamp::date, - sq4.device_id, - sq4.part ) sq5 - GROUP BY sq5.date, - sq5.device_id - ON CONFLICT DO NOTHING; - """.format(self.aircraft_table) - - self.cur.execute(query) - self.conn.commit() - - def create_gaps2d(self): - query = """ - INSERT INTO gaps2d(date, device_id, path) - SELECT sq3.date, - sq3.device_id, - ST_Collect(sq3.path) - FROM ( - SELECT - sq2.t1::DATE AS date, - sq2.d1 device_id, - ST_MakeLine(sq2.l1, sq2.l2) path - FROM - ( - SELECT sq.timestamp t1, - LAG(sq.timestamp) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) t2, - sq.location l1, - LAG(sq.location) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) l2, - sq.device_id d1, - LAG(sq.device_id) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) d2, - sq.agl a1, - LAG(sq.agl) over ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) a2 - FROM - ( - SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location, agl - FROM {} - ORDER BY device_id, timestamp, error_count - ) sq - ) sq2 - WHERE EXTRACT(epoch FROM sq2.t1 - sq2.t2) > 300 - AND ST_DistanceSphere(sq2.l1, sq2.l2) / EXTRACT(epoch FROM sq2.t1 - sq2.t2) BETWEEN 15 AND 50 - AND sq2.a1 > 300 AND sq2.a2 > 300 - ) sq3 - GROUP BY sq3.date, sq3.device_id - ON CONFLICT DO NOTHING; - """.format(self.aircraft_table) - - self.cur.execute(query) - self.conn.commit() - - -def convert(sourcefile, datestr, saver): - from ogn.gateway.process import string_to_message - from ogn.gateway.process_tools import AIRCRAFT_BEACON_TYPES, RECEIVER_BEACON_TYPES - from datetime import datetime - - fin = open_file(sourcefile) - - # get total lines of the input file - total_lines = 0 - for line in fin: - total_lines += 1 - fin.seek(0) - - current_line = 0 - steps = 100000 - reference_date = datetime.strptime(datestr + ' 12:00:00', '%Y-%m-%d %H:%M:%S') - - pbar = tqdm(fin, total=total_lines) - for line in pbar: - pbar.set_description('Importing {}'.format(sourcefile)) - - current_line += 1 - if current_line % steps == 0: - saver.flush() - - message = string_to_message(line.strip(), reference_date=reference_date) - if message is None: - continue - - dictfilt = lambda x, y: dict([(i, x[i]) for i in x if i in set(y)]) - - try: - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: - message = dictfilt(message, ('beacon_type', 'aprs_type', 'location_wkt', 'altitude', 'name', 'dstcall', 'relay', 'receiver_name', 'timestamp', 'track', 'ground_speed', - 'address_type', 'aircraft_type', 'stealth', 'address', 'climb_rate', 'turn_rate', 'signal_quality', 'error_count', 'frequency_offset', 'gps_quality_horizontal', 'gps_quality_vertical', 'software_version', 'hardware_version', 'real_address', 'signal_power', - 'distance', 'radial', 'quality', 'agl', 'location_mgrs', 'location_mgrs_short', - 'receiver_id', 'device_id')) - - beacon = AircraftBeacon(**message) - elif message['beacon_type'] in RECEIVER_BEACON_TYPES: - if 'rec_crystal_correction' in message: - del message['rec_crystal_correction'] - del message['rec_crystal_correction_fine'] - beacon = ReceiverBeacon(**message) - saver.add(beacon) - except Exception as e: - print(e) - - saver.flush() - fin.close() - - -@manager.command -def file_import(path): - """Import APRS logfiles into separate logfile tables.""" - - import os - import re - - # Get Filepaths and dates to import - results = list() - for (root, dirs, files) in os.walk(path): - for file in sorted(files): - match = re.match('OGN_log\.txt_([0-9]{4}\-[0-9]{2}\-[0-9]{2})\.gz$', file) - if match: - results.append({'filepath': os.path.join(root, file), - 'datestr': match.group(1)}) - - with LogfileDbSaver() as saver: - already_imported = saver.get_datestrs() - - results = list(filter(lambda x: x['datestr'] not in already_imported, results)) - - pbar = tqdm(results) - for result in pbar: - filepath = result['filepath'] - datestr = result['datestr'] - pbar.set_description("Importing data for {}".format(datestr)) - - saver.set_datestr(datestr) - saver.create_tables() - convert(filepath, datestr, saver) - saver.add_missing_devices() - saver.add_missing_receivers() - - -@manager.command -def update(): - """Update beacons (add foreign keys, compute distance, bearing, ags, etc.) in separate logfile tables.""" - - with LogfileDbSaver() as saver: - datestrs = saver.get_datestrs(no_index_only=True) - pbar = tqdm(datestrs) - for datestr in pbar: - pbar.set_description("Updating relations for {}".format(datestr)) - saver.set_datestr(datestr) - saver.update_receiver_location() - saver.update_aircraft_beacons() - saver.update_receiver_location() - saver.create_indices() - - -@manager.command -def transfer(start=None, end=None): - """Transfer beacons from separate logfile tables to beacon table.""" - - with LogfileDbSaver() as saver: - if start is not None and end is not None: - dates = get_database_days(start, end) - datestrs = [date.strftime('%Y_%m_%d') for date in dates] - else: - datestrs = saver.get_datestrs() - pbar = tqdm(datestrs) - for datestr in pbar: - pbar.set_description("Transfer beacons for {}".format(datestr)) - saver.set_datestr(datestr) - if not saver.is_transfered(): - saver.transfer_aircraft_beacons() - saver.transfer_receiver_beacons() - - -@manager.command -def create_flights2d(): - """Create complete flight traces from logfile tables.""" - - with LogfileDbSaver() as saver: - datestrs = saver.get_datestrs() - pbar = tqdm(datestrs) - for datestr in pbar: - pbar.set_description("Create Flights2D for {}".format(datestr)) - saver.set_datestr(datestr) - saver.create_flights2d() - - -@manager.command -def create_gaps2d(): - """Create 'gaps' from logfile tables.""" - - with LogfileDbSaver() as saver: - datestrs = saver.get_datestrs() - pbar = tqdm(datestrs) - for datestr in pbar: - pbar.set_description("Create Gaps2D for {}".format(datestr)) - saver.set_datestr(datestr) - saver.create_gaps2d() - - -@manager.command -def file_export(path): - """Export separate logfile tables to csv files. They can be used for fast bulk import with sql COPY command.""" - - import os - if not os.path.isdir(path): - print("'{}' is not a path. Exiting") - return - - with LogfileDbSaver() as saver: - datestrs = saver.get_datestrs() - datestrs = filter(lambda x: x.startswith('2018-12'), datestrs) - pbar = tqdm(datestrs) - for datestr in pbar: - pbar.set_description("Exporting data for {}".format(datestr)) - saver.set_datestr(datestr) - saver.export_to_path(path) - - -if __name__ == '__main__': - file_export() diff --git a/ogn/commands/flights.py b/ogn/commands/flights.py deleted file mode 100644 index 0d2e56f..0000000 --- a/ogn/commands/flights.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- - -from datetime import datetime - -from manager import Manager -from ogn.commands.dbutils import session -from ogn.commands.database import get_database_days -from tqdm import tqdm - -manager = Manager() - - -def compute_flights2d(session, date): - query = """ - INSERT INTO flights2d - ( - date, - device_id, - path, - path_simple - ) - SELECT sq5.date, - sq5.device_id, - st_collect(sq5.linestring order BY sq5.part) multilinestring, - st_collect(st_simplify(sq5.linestring ORDER BY sq5.part) simple_multilinestring - FROM ( - SELECT sq4.timestamp::date AS date, - sq4.device_id, - sq4.part, - st_makeline(sq4.location ORDER BY sq4.timestamp) linestring - FROM ( - SELECT sq3.timestamp, - sq3.location, - sq3.device_id, - sum(sq3.ping) OVER (partition BY sq3.timestamp::date, sq3.device_id ORDER BY sq3.timestamp) part - FROM ( - SELECT sq2.t1 AS timestamp, - sq2.l1 AS location, - sq2.d1 device_id, - CASE - WHEN sq2.t1 - sq2.t2 < interval'100s' AND ST_DistanceSphere(sq2.l1, sq2.l2) < 1000 THEN 0 - ELSE 1 - END AS ping - FROM ( - SELECT sq.timestamp t1, - lag(sq.timestamp) OVER (partition BY sq.device_id ORDER BY sq.timestamp) t2, - sq.location l1, - lag(sq.location) OVER (partition BY sq.device_id ORDER BY sq.timestamp) l2, - sq.device_id d1, - lag(sq.device_id) OVER (partition BY sq.device_id ORDER BY sq.timestamp) d2 - FROM ( - SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location - FROM aircraft_beacons - WHERE timestamp BETWEEN '{0} 00:00:00' AND '{0} 23:59:59' - ORDER BY device_id, timestamp, error_count - ) sq - ) sq2 - ) sq3 - ) sq4 - GROUP BY sq4.timestamp::date, sq4.device_id, sq4.part - ) sq5 - GROUP BY sq5.date, sq5.device_id - ON CONFLICT DO NOTHING; - """.format(date.strftime('%Y-%m-%d')) - session.execute(query) - session.commit() - - -@manager.command -def flights2d(start=None, end=None): - """Compute flights.""" - - days = get_database_days(start, end) - - pbar = tqdm(days) - for single_date in pbar: - pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) - result = compute_flights2d(session=session, date=single_date) diff --git a/ogn/commands/stats.py b/ogn/commands/stats.py deleted file mode 100644 index 13991f3..0000000 --- a/ogn/commands/stats.py +++ /dev/null @@ -1,57 +0,0 @@ -from datetime import datetime -from tqdm import tqdm -from manager import Manager -from ogn.commands.dbutils import session -from ogn.commands.database import get_database_days - -from ogn.collect.stats import create_device_stats, create_receiver_stats, create_relation_stats,\ - update_qualities, update_receivers as update_receivers_command, update_devices as update_devices_command,\ - update_device_stats_jumps - -from ogn.collect.ognrange import create_receiver_coverage - -manager = Manager() - - -@manager.command -def create(start=None, end=None): - """Create DeviceStats, ReceiverStats and RelationStats.""" - - days = get_database_days(start, end) - - pbar = tqdm(days) - for single_date in pbar: - pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) - result = create_device_stats(session=session, date=single_date) - result = update_device_stats_jumps(session=session, date=single_date) - result = create_receiver_stats(session=session, date=single_date) - result = create_relation_stats(session=session, date=single_date) - result = update_qualities(session=session, date=single_date) - - -@manager.command -def update_receivers(): - """Update receivers with data from stats.""" - - result = update_receivers_command(session=session) - print(result) - - -@manager.command -def update_devices(): - """Update devices with data from stats.""" - - result = update_devices_command(session=session) - print(result) - - -@manager.command -def create_ognrange(start=None, end=None): - """Create stats for Melissas ognrange.""" - - days = get_database_days(start, end) - - pbar = tqdm(days) - for single_date in pbar: - pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) - result = create_receiver_coverage(session=session, date=single_date) diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py deleted file mode 100644 index 2093aac..0000000 --- a/ogn/gateway/manage.py +++ /dev/null @@ -1,55 +0,0 @@ -import logging - -from manager import Manager -from ogn.client import AprsClient -from ogn.gateway.process import string_to_message -from datetime import datetime -from ogn.gateway.process_tools import DbSaver -from ogn.commands.dbutils import session - -manager = Manager() - -logging_formatstr = '%(asctime)s - %(levelname).4s - %(name)s - %(message)s' -log_levels = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'] - -saver = DbSaver(session=session) - - -def asdf(raw_string): - message = string_to_message(raw_string, reference_date=datetime.utcnow()) - if message is not None: - saver.add_message(message) - else: - print(message) - - -@manager.command -def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): - """Run the aprs client.""" - - # User input validation - if len(aprs_user) < 3 or len(aprs_user) > 9: - print('aprs_user must be a string of 3-9 characters.') - return - if loglevel not in log_levels: - print('loglevel must be an element of {}.'.format(log_levels)) - return - - # Enable logging - log_handlers = [logging.StreamHandler()] - if logfile: - log_handlers.append(logging.FileHandler(logfile)) - logging.basicConfig(format=logging_formatstr, level=loglevel, handlers=log_handlers) - - print('Start ogn gateway') - client = AprsClient(aprs_user) - client.connect() - - try: - client.run(callback=asdf, autoreconnect=True) - except KeyboardInterrupt: - print('\nStop ogn gateway') - - saver.flush() - client.disconnect() - logging.shutdown() diff --git a/ogn/gateway/process.py b/ogn/gateway/process.py deleted file mode 100644 index fc68ee9..0000000 --- a/ogn/gateway/process.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging - -from mgrs import MGRS - -from ogn.commands.dbutils import session -from ogn.model import Location -from ogn.parser import parse, ParseError -from ogn.gateway.process_tools import DbSaver, AIRCRAFT_BEACON_TYPES, RECEIVER_BEACON_TYPES - - -logger = logging.getLogger(__name__) -myMGRS = MGRS() - - -def _replace_lonlat_with_wkt(message): - latitude = message['latitude'] - longitude = message['longitude'] - - location = Location(longitude, latitude) - message['location_wkt'] = location.to_wkt() - location_mgrs = myMGRS.toMGRS(latitude, longitude).decode('utf-8') - message['location_mgrs'] = location_mgrs - message['location_mgrs_short'] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] - del message['latitude'] - del message['longitude'] - return message - - -def string_to_message(raw_string, reference_date): - global receivers - - try: - message = parse(raw_string, reference_date) - except NotImplementedError as e: - logger.w('No parser implemented for message: {}'.format(raw_string)) - return None - except ParseError as e: - logger.error('Parsing error with message: {}'.format(raw_string)) - return None - except TypeError as e: - logger.error('TypeError with message: {}'.format(raw_string)) - return None - except Exception as e: - logger.error(raw_string) - logger.error(e) - return None - - # update reference receivers and distance to the receiver - if message['aprs_type'] == 'position': - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES + RECEIVER_BEACON_TYPES: - message = _replace_lonlat_with_wkt(message) - - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and 'gps_quality' in message: - if message['gps_quality'] is not None and 'horizontal' in message['gps_quality']: - message['gps_quality_horizontal'] = message['gps_quality']['horizontal'] - message['gps_quality_vertical'] = message['gps_quality']['vertical'] - del message['gps_quality'] - - # update raw_message - message['raw_message'] = raw_string - - return message - - -saver = DbSaver(session=session) - - -def process_raw_message(raw_message, reference_date=None, saver=saver): - logger.debug('Received message: {}'.format(raw_message)) - message = string_to_message(raw_message, reference_date) - saver.add_message(message) diff --git a/ogn/gateway/process_tools.py b/ogn/gateway/process_tools.py deleted file mode 100644 index c07a8c3..0000000 --- a/ogn/gateway/process_tools.py +++ /dev/null @@ -1,126 +0,0 @@ -from datetime import datetime, timedelta -from ogn.model import AircraftBeacon, ReceiverBeacon -from ogn.collect.database import upsert - -# define message types we want to proceed -AIRCRAFT_BEACON_TYPES = ['aprs_aircraft', 'flarm', 'tracker', 'fanet', 'lt24', 'naviter', 'skylines', 'spider', 'spot'] -RECEIVER_BEACON_TYPES = ['aprs_receiver', 'receiver'] - -# define fields we want to proceed -BEACON_KEY_FIELDS = ['name', 'receiver_name', 'timestamp'] -AIRCRAFT_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'track', 'ground_speed', 'address_type', 'aircraft_type', 'stealth', 'address', 'climb_rate', 'turn_rate', 'signal_quality', 'error_count', 'frequency_offset', 'gps_quality_horizontal', 'gps_quality_vertical', 'software_version', 'hardware_version', 'real_address', 'signal_power', 'distance', 'radial', 'quality', 'location_mgrs', 'location_mgrs_short', 'agl', 'receiver_id', 'device_id'] -RECEIVER_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'version', 'platform', 'cpu_load', 'free_ram', 'total_ram', 'ntp_error', 'rt_crystal_correction', 'voltage', 'amperage', 'cpu_temp', 'senders_visible', 'senders_total', 'rec_input_noise', 'senders_signal', 'senders_messages', 'good_senders_signal', 'good_senders', 'good_and_bad_senders'] - - -class DummyMerger: - def __init__(self, callback): - self.callback = callback - - def add_message(self, message): - self.callback.add_message(message) - - def flush(self): - pass - - -class DbSaver: - def __init__(self, session): - self.session = session - self.aircraft_message_map = dict() - self.receiver_message_map = dict() - self.last_commit = datetime.utcnow() - - def _put_in_map(self, message, my_map): - key = message['name'] + message['receiver_name'] + message['timestamp'].strftime('%s') - - if key in my_map: - other = my_map[key] - merged = {k: message[k] if message[k] is not None else other[k] for k in message.keys()} - my_map[key] = merged - else: - my_map[key] = message - - def add_message(self, message): - if message is None or ('raw_message' in message and message['raw_message'][0] == '#') or 'beacon_type' not in message: - return - - if 'location_wkt' in message: - message['location'] = message.pop('location_wkt') # total_time_wasted_here = 3 - - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: - even_messages = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS} - self._put_in_map(message=even_messages, my_map=self.aircraft_message_map) - elif message['beacon_type'] in RECEIVER_BEACON_TYPES: - even_messages = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS} - self._put_in_map(message=even_messages, my_map=self.receiver_message_map) - else: - print("Ignore beacon_type: {}".format(message['beacon_type'])) - return - - elapsed_time = datetime.utcnow() - self.last_commit - if elapsed_time >= timedelta(seconds=5): - self.flush() - - def flush(self): - if len(self.aircraft_message_map) > 0: - messages = list(self.aircraft_message_map.values()) - upsert(session=self.session, model=AircraftBeacon, rows=messages, update_cols=AIRCRAFT_BEACON_FIELDS) - if len(self.receiver_message_map) > 0: - messages = list(self.receiver_message_map.values()) - upsert(session=self.session, model=ReceiverBeacon, rows=messages, update_cols=RECEIVER_BEACON_FIELDS) - self.session.commit() - - self.aircraft_message_map = dict() - self.receiver_message_map = dict() - self.last_commit = datetime.utcnow() - - -class DummySaver: - def add_message(self, message): - print(message) - - def flush(self): - print("========== flush ==========") - - -import os, gzip, csv - - -class FileSaver: - def __init__(self): - self.aircraft_messages = list() - self.receiver_messages = list() - - def open(self, path, reference_date_string): - aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz') - receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz') - - if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename): - self.fout_ab = gzip.open(aircraft_beacon_filename, 'wt') - self.fout_rb = gzip.open(receiver_beacon_filename, 'wt') - else: - raise FileExistsError - - self.aircraft_writer = csv.writer(self.fout_ab, delimiter=',') - self.aircraft_writer.writerow(AircraftBeacon.get_columns()) - - self.receiver_writer = csv.writer(self.fout_rb, delimiter=',') - self.receiver_writer.writerow(ReceiverBeacon.get_columns()) - - return 1 - - def add_message(self, beacon): - if isinstance(beacon, AircraftBeacon): - self.aircraft_messages.append(beacon.get_values()) - elif isinstance(beacon, ReceiverBeacon): - self.receiver_messages.append(beacon.get_values()) - - def flush(self): - self.aircraft_writer.writerows(self.aircraft_messages) - self.receiver_writer.writerows(self.receiver_messages) - self.aircraft_messages = list() - self.receiver_messages = list() - - def close(self): - self.fout_ab.close() - self.fout_rb.close() diff --git a/ogn/model/airport.py b/ogn/model/airport.py deleted file mode 100644 index 0dd305a..0000000 --- a/ogn/model/airport.py +++ /dev/null @@ -1,38 +0,0 @@ -from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, Float, SmallInteger - -from .base import Base - - -class Airport(Base): - __tablename__ = "airports" - - id = Column(Integer, primary_key=True) - - location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Float(precision=2)) - - name = Column(String, index=True) - code = Column(String(6)) - country_code = Column(String(2)) - style = Column(SmallInteger) - description = Column(String) - runway_direction = Column(SmallInteger) - runway_length = Column(SmallInteger) - frequency = Column(Float(precision=2)) - - border = Column('border', Geometry('POLYGON', srid=4326)) - - def __repr__(self): - return "" % ( - self.name, - self.code, - self.country_code, - self.style, - self.description, - self.location_wkt.latitude if self.location_wkt else None, - self.location_wkt.longitude if self.location_wkt else None, - self.altitude, - self.runway_direction, - self.runway_length, - self.frequency) diff --git a/ogn/model/base.py b/ogn/model/base.py deleted file mode 100644 index 00ea8e1..0000000 --- a/ogn/model/base.py +++ /dev/null @@ -1,4 +0,0 @@ -from sqlalchemy.ext.declarative import declarative_base - - -Base = declarative_base() diff --git a/ogn/model/country.py b/ogn/model/country.py deleted file mode 100644 index 0276e13..0000000 --- a/ogn/model/country.py +++ /dev/null @@ -1,39 +0,0 @@ -from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, Integer, Float, SmallInteger, BigInteger - -from .base import Base - - -class Country(Base): - __tablename__ = "countries" - - gid = Column(Integer, primary_key=True) - - fips = Column(String(2)) - iso2 = Column(String(2)) - iso3 = Column(String(3)) - - un = Column(SmallInteger) - name = Column(String(50)) - area = Column(Integer) - pop2005 = Column(BigInteger) - region = Column(SmallInteger) - subregion = Column(SmallInteger) - lon = Column(Float) - lat = Column(Float) - - geom = Column('geom', Geometry('MULTIPOLYGON', srid=4326)) - - def __repr__(self): - return "" % ( - self.fips, - self.iso2, - self.iso3, - self.un, - self.name, - self.area, - self.pop2005, - self.region, - self.subregion, - self.lon, - self.lat) diff --git a/ogn/model/device.py b/ogn/model/device.py deleted file mode 100644 index 74d89b6..0000000 --- a/ogn/model/device.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlalchemy import Column, Integer, String, Float, Boolean, SmallInteger, DateTime - -from .base import Base - - -class Device(Base): - __tablename__ = 'devices' - - id = Column(Integer, primary_key=True) - - #address = Column(String(6), index=True) - address = Column(String, index=True) - firstseen = Column(DateTime, index=True) - lastseen = Column(DateTime, index=True) - aircraft_type = Column(SmallInteger, index=True) - stealth = Column(Boolean) - software_version = Column(Float(precision=2)) - hardware_version = Column(SmallInteger) - real_address = Column(String(6)) - - def __repr__(self): - return "" % ( - self.address, - self.aircraft_type, - self.stealth, - self.software_version, - self.hardware_version, - self.real_address) diff --git a/ogn/model/device_info.py b/ogn/model/device_info.py deleted file mode 100644 index f415f2c..0000000 --- a/ogn/model/device_info.py +++ /dev/null @@ -1,37 +0,0 @@ -from sqlalchemy import Column, Integer, String, Boolean, SmallInteger, ForeignKey -from sqlalchemy.orm import relationship, backref - -from .base import Base - - -class DeviceInfo(Base): - __tablename__ = 'device_infos' - - id = Column(Integer, primary_key=True) - address_type = None - #address = Column(String(6), index=True) - address = Column(String, index=True) - aircraft = Column(String) - registration = Column(String(7)) - competition = Column(String(3)) - tracked = Column(Boolean) - identified = Column(Boolean) - aircraft_type = Column(SmallInteger) - - address_origin = Column(SmallInteger) - - # Relations - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) - device = relationship('Device', foreign_keys=[device_id], backref=backref('infos', order_by='DeviceInfo.address_origin.asc()')) - - def __repr__(self): - return "" % ( - self.address_type, - self.address, - self.aircraft, - self.registration, - self.competition, - self.tracked, - self.identified, - self.aircraft_type, - self.address_origin) diff --git a/ogn/model/device_stats.py b/ogn/model/device_stats.py deleted file mode 100644 index be56e7f..0000000 --- a/ogn/model/device_stats.py +++ /dev/null @@ -1,56 +0,0 @@ -from sqlalchemy import Column, Integer, Date, DateTime, Float, ForeignKey, SmallInteger, Boolean, String, Index -from sqlalchemy.orm import relationship, backref - -from .base import Base - - -class DeviceStats(Base): - __tablename__ = "device_stats" - - id = Column(Integer, primary_key=True) - - date = Column(Date) - - # Static data - firstseen = Column(DateTime) - lastseen = Column(DateTime) - aircraft_type = Column(SmallInteger) - stealth = Column(Boolean) - software_version = Column(Float(precision=2)) - hardware_version = Column(SmallInteger) - real_address = Column(String(6)) - - # Statistic data - max_altitude = Column(Float(precision=2)) - receiver_count = Column(SmallInteger) - aircraft_beacon_count = Column(Integer) - jumps = Column(SmallInteger) - ambiguous = Column(Boolean) - quality = Column(Float(precision=2)) - - # Relation statistic data - quality_offset = Column(Float(precision=2)) - - # Ranking data - max_altitude_ranking_worldwide = Column(Integer) - max_altitude_ranking_country = Column(Integer) - receiver_count_ranking_worldwide = Column(Integer) - receiver_count_ranking_country = Column(Integer) - aircraft_beacon_count_ranking_worldwide = Column(Integer) - aircraft_beacon_count_ranking_country = Column(Integer) - quality_ranking_worldwide = Column(Integer) - quality_ranking_country = Column(Integer) - - # Relations - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) - device = relationship('Device', foreign_keys=[device_id], backref=backref('stats', order_by='DeviceStats.date.asc()')) - - def __repr__(self): - return "" % ( - self.date, - self.receiver_count, - self.aircraft_beacon_count, - self.max_altitude) - - -Index('ix_device_stats_date_device_id', DeviceStats.date, DeviceStats.device_id) diff --git a/ogn/model/flights2d.py b/ogn/model/flights2d.py deleted file mode 100644 index 030ef4a..0000000 --- a/ogn/model/flights2d.py +++ /dev/null @@ -1,28 +0,0 @@ -from geoalchemy2.types import Geometry -from sqlalchemy import Column, Integer, Date, Index, ForeignKey -from sqlalchemy.orm import relationship - -from .base import Base - - -class Flight2D(Base): - __tablename__ = "flights2d" - - date = Column(Date, primary_key=True) - - path_wkt = Column('path', Geometry('MULTILINESTRING', srid=4326)) - path_simple_wkt = Column('path_simple', Geometry('MULTILINESTRING', srid=4326)) # this is the path simplified with ST_Simplify(path, 0.0001) - - # Relations - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True) - device = relationship('Device', foreign_keys=[device_id], backref='flights2d') - - def __repr__(self): - return "" % ( - self.date, - self.path_wkt, - self.path_simple_wkt) - - -Index('ix_flights2d_date_device_id', Flight2D.date, Flight2D.device_id) -#Index('ix_flights2d_date_path', Flight2D.date, Flight2D.path_wkt) --> CREATE INDEX ix_flights2d_date_path ON flights2d USING GIST("date", path) diff --git a/ogn/model/logbook.py b/ogn/model/logbook.py deleted file mode 100644 index 9f2856c..0000000 --- a/ogn/model/logbook.py +++ /dev/null @@ -1,36 +0,0 @@ -from sqlalchemy import Integer, SmallInteger, Float, DateTime, Column, ForeignKey, case, null -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import relationship, backref - -from .base import Base - - -class Logbook(Base): - __tablename__ = 'logbook' - - id = Column(Integer, primary_key=True) - - reftime = Column(DateTime, index=True) - takeoff_timestamp = Column(DateTime) - takeoff_track = Column(SmallInteger) - landing_timestamp = Column(DateTime) - landing_track = Column(SmallInteger) - max_altitude = Column(Float(precision=2)) - - # Relations - takeoff_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True) - takeoff_airport = relationship('Airport', foreign_keys=[takeoff_airport_id]) - - landing_airport_id = Column(Integer, ForeignKey('airports.id', ondelete='CASCADE'), index=True) - landing_airport = relationship('Airport', foreign_keys=[landing_airport_id]) - - device_id = Column(Integer, ForeignKey('devices.id', ondelete='CASCADE'), index=True) - device = relationship('Device', foreign_keys=[device_id], backref=backref('logbook', order_by='Logbook.reftime')) - - @hybrid_property - def duration(self): - return None if (self.landing_timestamp is None or self.takeoff_timestamp is None) else self.landing_timestamp - self.takeoff_timestamp - - @duration.expression - def duration(cls): - return case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != null() and cls.takeoff_timestamp != null()) diff --git a/ogn/model/receiver.py b/ogn/model/receiver.py deleted file mode 100644 index 51b5552..0000000 --- a/ogn/model/receiver.py +++ /dev/null @@ -1,34 +0,0 @@ -from geoalchemy2.shape import to_shape -from geoalchemy2.types import Geometry -from sqlalchemy import Column, Float, String, Integer, DateTime, ForeignKey -from sqlalchemy.orm import relationship, backref - -from .base import Base -from .geo import Location - - -class Receiver(Base): - __tablename__ = "receivers" - - id = Column(Integer, primary_key=True) - - location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Float(precision=2)) - - name = Column(String(9), index=True) - firstseen = Column(DateTime, index=True) - lastseen = Column(DateTime, index=True) - version = Column(String) - platform = Column(String) - - # Relations - country_id = Column(Integer, ForeignKey('countries.gid', ondelete='SET NULL'), index=True) - country = relationship('Country', foreign_keys=[country_id], backref=backref('receivers', order_by='Receiver.name.asc()')) - - @property - def location(self): - if self.location_wkt is None: - return None - - coords = to_shape(self.location_wkt) - return Location(lat=coords.y, lon=coords.x) diff --git a/ogn/model/receiver_coverage.py b/ogn/model/receiver_coverage.py deleted file mode 100644 index fa8e3a1..0000000 --- a/ogn/model/receiver_coverage.py +++ /dev/null @@ -1,26 +0,0 @@ -from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index -from sqlalchemy.orm import relationship, backref - -from .base import Base - - -class ReceiverCoverage(Base): - __tablename__ = "receiver_coverages" - - location_mgrs_short = Column(String(9), primary_key=True) - date = Column(Date, primary_key=True) - - max_signal_quality = Column(Float) - max_altitude = Column(Float(precision=2)) - min_altitude = Column(Float(precision=2)) - aircraft_beacon_count = Column(Integer) - - device_count = Column(SmallInteger) - - # Relations - receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), primary_key=True) - receiver = relationship('Receiver', foreign_keys=[receiver_id], backref=backref('receiver_coverages', order_by='ReceiverCoverage.date.asc()')) - - -Index('ix_receiver_coverages_date_receiver_id', ReceiverCoverage.date, ReceiverCoverage.receiver_id) -Index('ix_receiver_coverages_receiver_id_date', ReceiverCoverage.receiver_id, ReceiverCoverage.date) diff --git a/ogn/model/receiver_stats.py b/ogn/model/receiver_stats.py deleted file mode 100644 index 2dcedae..0000000 --- a/ogn/model/receiver_stats.py +++ /dev/null @@ -1,43 +0,0 @@ -from sqlalchemy import Column, Integer, SmallInteger, Date, Float, ForeignKey, DateTime, String, Index -from sqlalchemy.orm import relationship, backref -from geoalchemy2.types import Geometry - -from .base import Base - - -class ReceiverStats(Base): - __tablename__ = "receiver_stats" - - id = Column(Integer, primary_key=True) - - date = Column(Date) - - # Static data - firstseen = Column(DateTime, index=True) - lastseen = Column(DateTime, index=True) - location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Float(precision=2)) - version = Column(String) - platform = Column(String) - - # Statistic data - aircraft_beacon_count = Column(Integer) - aircraft_count = Column(SmallInteger) - max_distance = Column(Float) - quality = Column(Float(precision=2)) - - # Relation statistic data - quality_offset = Column(Float(precision=2)) - - # Ranking data - aircraft_beacon_count_ranking = Column(SmallInteger) - aircraft_count_ranking = Column(SmallInteger) - max_distance_ranking = Column(SmallInteger) - quality_ranking = Column(Integer) - - # Relations - receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True) - receiver = relationship('Receiver', foreign_keys=[receiver_id], backref=backref('stats', order_by='ReceiverStats.date.asc()')) - - -Index('ix_receiver_stats_date_receiver_id', ReceiverStats.date, ReceiverStats.receiver_id) diff --git a/ogn/model/relation_stats.py b/ogn/model/relation_stats.py deleted file mode 100644 index 1e8cd5d..0000000 --- a/ogn/model/relation_stats.py +++ /dev/null @@ -1,32 +0,0 @@ -from sqlalchemy import Column, Integer, Date, Float, ForeignKey, Index -from sqlalchemy.orm import relationship - -from .base import Base - - -class RelationStats(Base): - __tablename__ = "relation_stats" - - id = Column(Integer, primary_key=True) - - date = Column(Date) - - # Statistic data - quality = Column(Float(precision=2)) - beacon_count = Column(Integer) - - # Relations - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True) - device = relationship('Device', foreign_keys=[device_id], backref='relation_stats') - receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL'), index=True) - receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='relation_stats') - - def __repr__(self): - return "" % ( - self.date, - self.quality, - self.beacon_count) - - -Index('ix_relation_stats_date_device_id', RelationStats.date, RelationStats.device_id, RelationStats.receiver_id) -Index('ix_relation_stats_date_receiver_id', RelationStats.date, RelationStats.receiver_id, RelationStats.device_id) diff --git a/ogn/model/takeoff_landing.py b/ogn/model/takeoff_landing.py deleted file mode 100644 index 91c2ab1..0000000 --- a/ogn/model/takeoff_landing.py +++ /dev/null @@ -1,19 +0,0 @@ -from sqlalchemy import Boolean, Column, Integer, SmallInteger, DateTime, ForeignKey -from sqlalchemy.orm import relationship - -from .base import Base - - -class TakeoffLanding(Base): - __tablename__ = 'takeoff_landings' - - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True) - airport_id = Column(Integer, ForeignKey('airports.id', ondelete='SET NULL'), primary_key=True) - timestamp = Column(DateTime, primary_key=True) - - is_takeoff = Column(Boolean) - track = Column(SmallInteger) - - # Relations - airport = relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings') - device = relationship('Device', foreign_keys=[device_id], backref='takeoff_landings', order_by='TakeoffLanding.timestamp') diff --git a/ogn_python/__init__.py b/ogn_python/__init__.py new file mode 100644 index 0000000..8478c04 --- /dev/null +++ b/ogn_python/__init__.py @@ -0,0 +1,18 @@ +from flask import Flask +from flask_bootstrap import Bootstrap +from flask_sqlalchemy import SQLAlchemy + +from ogn_python.navigation import nav + +# Initialize Flask +app = Flask(__name__) +app.config.from_object('config.default') + +# Bootstrap +bootstrap = Bootstrap(app) + +# Sqlalchemy +db = SQLAlchemy(app) + +# Navigation +nav.init_app(app) diff --git a/ogn_python/app.py b/ogn_python/app.py new file mode 100644 index 0000000..92fb3a3 --- /dev/null +++ b/ogn_python/app.py @@ -0,0 +1,6 @@ +from ogn_python import app +from ogn_python import routes +from ogn_python import commands + +if __name__ == '__main__': + app.run() diff --git a/ogn/backend/__init__.py b/ogn_python/backend/__init__.py similarity index 100% rename from ogn/backend/__init__.py rename to ogn_python/backend/__init__.py diff --git a/ogn/backend/liveglidernet.py b/ogn_python/backend/liveglidernet.py similarity index 98% rename from ogn/backend/liveglidernet.py rename to ogn_python/backend/liveglidernet.py index 2ae132d..d23ee9e 100644 --- a/ogn/backend/liveglidernet.py +++ b/ogn_python/backend/liveglidernet.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta, timezone, date from sqlalchemy import func, and_, between, case -from ogn.model import AircraftBeacon, Device, Receiver +from ogn_python.model import AircraftBeacon, Device, Receiver def utc_to_local(utc_dt): diff --git a/ogn/backend/ognrange.py b/ogn_python/backend/ognrange.py similarity index 96% rename from ogn/backend/ognrange.py rename to ogn_python/backend/ognrange.py index 870bbc4..bf2732e 100644 --- a/ogn/backend/ognrange.py +++ b/ogn_python/backend/ognrange.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from sqlalchemy import func, case from sqlalchemy.sql.expression import label -from ogn.model import Receiver +from ogn_python.model import Receiver def alchemyencoder(obj): diff --git a/ogn/collect/__init__.py b/ogn_python/collect/__init__.py similarity index 100% rename from ogn/collect/__init__.py rename to ogn_python/collect/__init__.py diff --git a/ogn/collect/celery.py b/ogn_python/collect/celery.py similarity index 100% rename from ogn/collect/celery.py rename to ogn_python/collect/celery.py diff --git a/ogn_python/collect/database.py b/ogn_python/collect/database.py new file mode 100644 index 0000000..d5de55b --- /dev/null +++ b/ogn_python/collect/database.py @@ -0,0 +1,87 @@ +from celery.utils.log import get_task_logger + +from sqlalchemy import distinct +from sqlalchemy.sql import null, and_, func, not_, case +from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects.postgresql import insert + +from ogn_python.collect.celery import app +from ogn_python.model import Country, DeviceInfo, DeviceInfoOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn_python.utils import get_ddb, get_flarmnet + + +logger = get_task_logger(__name__) + + +def compile_query(query): + """Via http://nicolascadou.com/blog/2014/01/printing-actual-sqlalchemy-queries""" + compiler = query.compile if not hasattr(query, 'statement') else query.statement.compile + return compiler(dialect=postgresql.dialect()) + + +def upsert(session, model, rows, update_cols): + """Insert rows in model. On conflicting update columns if new value IS NOT NULL.""" + + table = model.__table__ + + stmt = insert(table).values(rows) + + on_conflict_stmt = stmt.on_conflict_do_update( + index_elements=table.primary_key.columns, + set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols}, + ) + + # print(compile_query(on_conflict_stmt)) + session.execute(on_conflict_stmt) + + +def update_device_infos(session, address_origin, path=None): + if address_origin == DeviceInfoOrigin.flarmnet: + device_infos = get_flarmnet(fln_file=path) + else: + device_infos = get_ddb(csv_file=path) + + session.query(DeviceInfo) \ + .filter(DeviceInfo.address_origin == address_origin) \ + .delete(synchronize_session='fetch') + session.commit() + + for device_info in device_infos: + device_info.address_origin = address_origin + + session.bulk_save_objects(device_infos) + session.commit() + + return len(device_infos) + + +@app.task +def import_ddb(session=None): + """Import registered devices from the DDB.""" + + if session is None: + session = app.session + + logger.info("Import registered devices fom the DDB...") + counter = update_device_infos(session, DeviceInfoOrigin.ogn_ddb) + logger.info("Imported {} devices.".format(counter)) + + return "Imported {} devices.".format(counter) + + +@app.task +def update_country_code(session=None): + """Update country code in receivers table if None.""" + + if session is None: + session = app.session + + update_receivers = session.query(Receiver) \ + .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom))) \ + .update({Receiver.country_id: Country.gid}, + synchronize_session='fetch') + + session.commit() + logger.info("Updated {} AircraftBeacons".format(update_receivers)) + + return "Updated country for {} Receivers".format(update_receivers) diff --git a/ogn/collect/logbook.py b/ogn_python/collect/logbook.py similarity index 98% rename from ogn/collect/logbook.py rename to ogn_python/collect/logbook.py index 309d69a..99dec3f 100644 --- a/ogn/collect/logbook.py +++ b/ogn_python/collect/logbook.py @@ -4,9 +4,9 @@ from sqlalchemy import and_, or_, insert, update, exists, between from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import true, false -from ogn.collect.celery import app -from ogn.model import TakeoffLanding, Logbook, AircraftBeacon -from ogn.utils import date_to_timestamps +from ogn_python.collect.celery import app +from ogn_python.model import TakeoffLanding, Logbook, AircraftBeacon +from ogn_python.utils import date_to_timestamps logger = get_task_logger(__name__) diff --git a/ogn/collect/ognrange.py b/ogn_python/collect/ognrange.py similarity index 96% rename from ogn/collect/ognrange.py rename to ogn_python/collect/ognrange.py index 17a0d02..61d02fb 100644 --- a/ogn/collect/ognrange.py +++ b/ogn_python/collect/ognrange.py @@ -4,9 +4,9 @@ from sqlalchemy import Date from sqlalchemy import and_, insert, update, exists, between from sqlalchemy.sql import func, null -from ogn.collect.celery import app -from ogn.model import AircraftBeacon, ReceiverCoverage -from ogn.utils import date_to_timestamps +from ogn_python.collect.celery import app +from ogn_python.model import AircraftBeacon, ReceiverCoverage +from ogn_python.utils import date_to_timestamps logger = get_task_logger(__name__) diff --git a/ogn/collect/stats.py b/ogn_python/collect/stats.py similarity index 84% rename from ogn/collect/stats.py rename to ogn_python/collect/stats.py index f9d8b2e..0003219 100644 --- a/ogn/collect/stats.py +++ b/ogn_python/collect/stats.py @@ -1,17 +1,20 @@ from celery.utils.log import get_task_logger -from sqlalchemy import insert, distinct, between +from sqlalchemy import insert, distinct, between, literal from sqlalchemy.sql import null, and_, func, or_, update from sqlalchemy.sql.expression import case -from ogn.model import AircraftBeacon, DeviceStats, ReceiverStats, RelationStats, Receiver, Device +from ogn_python.model import AircraftBeacon, DeviceStats, Country, CountryStats, ReceiverStats, RelationStats, Receiver, Device from .celery import app -from ogn.model.receiver_beacon import ReceiverBeacon -from ogn.utils import date_to_timestamps +from ogn_python.model.receiver_beacon import ReceiverBeacon +from ogn_python.utils import date_to_timestamps logger = get_task_logger(__name__) +# 40dB@10km is enough for 640km +MAX_PLAUSIBLE_QUALITY = 40 + @app.task def create_device_stats(session=None, date=None): @@ -43,7 +46,7 @@ def create_device_stats(session=None, date=None): # Calculate stats, firstseen, lastseen and last values != NULL device_stats = session.query( distinct(sq.c.device_id).label('device_id'), - func.date(sq.c.timestamp).label('date'), + literal(date).label('date'), func.max(sq.c.dr) .over(partition_by=sq.c.device_id) .label('receiver_count'), @@ -53,6 +56,9 @@ def create_device_stats(session=None, date=None): func.count(sq.c.device_id) .over(partition_by=sq.c.device_id) .label('aircraft_beacon_count'), + func.first_value(sq.c.name) + .over(partition_by=sq.c.device_id, order_by=case([(sq.c.name == null(), None)], else_=sq.c.timestamp).asc().nullslast()) + .label('name'), func.first_value(sq.c.timestamp) .over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()) .label('firstseen'), @@ -78,7 +84,8 @@ def create_device_stats(session=None, date=None): # And insert them ins = insert(DeviceStats).from_select( - [DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.max_altitude, DeviceStats.aircraft_beacon_count, DeviceStats.firstseen, DeviceStats.lastseen, DeviceStats.aircraft_type, DeviceStats.stealth, + [DeviceStats.device_id, DeviceStats.date, DeviceStats.receiver_count, DeviceStats.max_altitude, DeviceStats.aircraft_beacon_count, DeviceStats.name, + DeviceStats.firstseen, DeviceStats.lastseen, DeviceStats.aircraft_type, DeviceStats.stealth, DeviceStats.software_version, DeviceStats.hardware_version, DeviceStats.real_address], device_stats) res = session.execute(ins) @@ -115,7 +122,7 @@ def create_receiver_stats(session=None, date=None): # Calculate stats, firstseen, lastseen and last values != NULL receiver_stats = session.query( distinct(sq.c.receiver_id).label('receiver_id'), - func.date(sq.c.timestamp).label('date'), + literal(date).label('date'), func.first_value(sq.c.timestamp) .over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()) .label('firstseen'), @@ -145,21 +152,20 @@ def create_receiver_stats(session=None, date=None): session.commit() logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) - # Update aircraft_beacon_count, aircraft_count and max_distance (without any error and max quality of 36dB@10km which is enough for 640km ... ) - aircraft_beacon_stats = session.query(func.date(AircraftBeacon.timestamp).label('date'), - AircraftBeacon.receiver_id, + # Update aircraft_beacon_count, aircraft_count and max_distance + aircraft_beacon_stats = session.query(AircraftBeacon.receiver_id, func.count(AircraftBeacon.timestamp).label('aircraft_beacon_count'), func.count(func.distinct(AircraftBeacon.device_id)).label('aircraft_count'), func.max(AircraftBeacon.distance).label('max_distance')) \ .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.error_count == 0, - AircraftBeacon.quality <= 40)) \ - .group_by(func.date(AircraftBeacon.timestamp), - AircraftBeacon.receiver_id) \ + AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY, + AircraftBeacon.relay == null())) \ + .group_by(AircraftBeacon.receiver_id) \ .subquery() upd = update(ReceiverStats) \ - .where(and_(ReceiverStats.date == aircraft_beacon_stats.c.date, + .where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id)) \ .values({'aircraft_beacon_count': aircraft_beacon_stats.c.aircraft_beacon_count, 'aircraft_count': aircraft_beacon_stats.c.aircraft_count, @@ -173,6 +179,39 @@ def create_receiver_stats(session=None, date=None): return "ReceiverStats for {}: {} deleted, {} inserted, {} updated".format(date, deleted_counter, insert_counter, update_counter) +@app.task +def create_country_stats(session=None, date=None): + if session is None: + session = app.session + + if not date: + logger.warn("A date is needed for calculating stats. Exiting") + return None + else: + (start, end) = date_to_timestamps(date) + + # First kill the stats for the selected date + deleted_counter = session.query(CountryStats) \ + .filter(CountryStats.date == date) \ + .delete() + + country_stats = session.query(literal(date), Country.gid, + func.count(AircraftBeacon.timestamp).label('aircraft_beacon_count'), \ + func.count(func.distinct(AircraftBeacon.receiver_id)).label('device_count')) \ + .filter(between(AircraftBeacon.timestamp, start, end)) \ + .filter(func.st_contains(Country.geom, AircraftBeacon.location)) \ + .group_by(Country.gid) \ + .subquery() + + # And insert them + ins = insert(CountryStats).from_select( + [CountryStats.date, CountryStats.country_id, CountryStats.aircraft_beacon_count, CountryStats.device_count], + country_stats) + res = session.execute(ins) + insert_counter = res.rowcount + session.commit() + + @app.task def update_device_stats_jumps(session=None, date=None): """Update device stats jumps.""" @@ -252,7 +291,7 @@ def create_relation_stats(session=None, date=None): # Calculate stats for selected day relation_stats = session.query( - func.date(AircraftBeacon.timestamp), + literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id, func.max(AircraftBeacon.quality), @@ -261,9 +300,9 @@ def create_relation_stats(session=None, date=None): .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.distance > 1000, AircraftBeacon.error_count == 0, - AircraftBeacon.quality <= 40, + AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY, AircraftBeacon.ground_speed > 10)) \ - .group_by(func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.receiver_id) \ + .group_by(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id) \ .subquery() # And insert them @@ -290,16 +329,14 @@ def update_qualities(session=None, date=None): return None # Calculate avg quality of devices - dev_sq = session.query(RelationStats.date, - RelationStats.device_id, + dev_sq = session.query(RelationStats.device_id, func.avg(RelationStats.quality).label('quality')) \ .filter(RelationStats.date == date) \ - .group_by(RelationStats.date, - RelationStats.device_id) \ + .group_by(RelationStats.device_id) \ .subquery() dev_upd = update(DeviceStats) \ - .where(and_(DeviceStats.date == dev_sq.c.date, + .where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)) \ .values({'quality': dev_sq.c.quality}) @@ -309,16 +346,14 @@ def update_qualities(session=None, date=None): logger.warn("Updated {} DeviceStats: quality".format(dev_update_counter)) # Calculate avg quality of receivers - rec_sq = session.query(RelationStats.date, - RelationStats.receiver_id, + rec_sq = session.query(RelationStats.receiver_id, func.avg(RelationStats.quality).label('quality')) \ .filter(RelationStats.date == date) \ - .group_by(RelationStats.date, - RelationStats.receiver_id) \ + .group_by(RelationStats.receiver_id) \ .subquery() rec_upd = update(ReceiverStats) \ - .where(and_(ReceiverStats.date == rec_sq.c.date, + .where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)) \ .values({'quality': rec_sq.c.quality}) @@ -328,18 +363,16 @@ def update_qualities(session=None, date=None): logger.warn("Updated {} ReceiverStats: quality".format(rec_update_counter)) # Calculate quality_offset of devices - dev_sq = session.query(RelationStats.date, - RelationStats.device_id, - (func.sum(RelationStats.beacon_count * (RelationStats.quality - ReceiverStats.quality)) / (func.sum(RelationStats.beacon_count))).label('quality_offset')) \ + dev_sq = session.query(RelationStats.device_id, + (func.sum(RelationStats.beacon_count * (RelationStats.quality - ReceiverStats.quality)) / (func.sum(RelationStats.beacon_count))).label('quality_offset')) \ .filter(RelationStats.date == date) \ .filter(and_(RelationStats.receiver_id == ReceiverStats.receiver_id, RelationStats.date == ReceiverStats.date)) \ - .group_by(RelationStats.date, - RelationStats.device_id) \ + .group_by(RelationStats.device_id) \ .subquery() dev_upd = update(DeviceStats) \ - .where(and_(DeviceStats.date == dev_sq.c.date, + .where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)) \ .values({'quality_offset': dev_sq.c.quality_offset}) @@ -349,18 +382,16 @@ def update_qualities(session=None, date=None): logger.warn("Updated {} DeviceStats: quality_offset".format(dev_update_counter)) # Calculate quality_offset of receivers - rec_sq = session.query(RelationStats.date, - RelationStats.receiver_id, - (func.sum(RelationStats.beacon_count * (RelationStats.quality - DeviceStats.quality)) / (func.sum(RelationStats.beacon_count))).label('quality_offset')) \ + rec_sq = session.query(RelationStats.receiver_id, + (func.sum(RelationStats.beacon_count * (RelationStats.quality - DeviceStats.quality)) / (func.sum(RelationStats.beacon_count))).label('quality_offset')) \ .filter(RelationStats.date == date) \ .filter(and_(RelationStats.device_id == DeviceStats.device_id, RelationStats.date == DeviceStats.date)) \ - .group_by(RelationStats.date, - RelationStats.receiver_id) \ + .group_by(RelationStats.receiver_id) \ .subquery() rec_upd = update(ReceiverStats) \ - .where(and_(ReceiverStats.date == rec_sq.c.date, + .where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)) \ .values({'quality_offset': rec_sq.c.quality_offset}) @@ -428,6 +459,9 @@ def update_devices(session=None): device_stats = session.query( distinct(DeviceStats.device_id).label('device_id'), + func.first_value(DeviceStats.name) + .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.name == null(), None)], else_=DeviceStats.date).desc().nullslast()) + .label('name'), func.first_value(DeviceStats.firstseen) .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.firstseen == null(), None)], else_=DeviceStats.date).asc().nullslast()) .label('firstseen'), @@ -454,7 +488,8 @@ def update_devices(session=None): upd = update(Device) \ .where(and_(Device.id == device_stats.c.device_id)) \ - .values({'firstseen': device_stats.c.firstseen, + .values({'name': device_stats.c.name, + 'firstseen': device_stats.c.firstseen, 'lastseen': device_stats.c.lastseen, 'aircraft_type': device_stats.c.aircraft_type, 'stealth': device_stats.c.stealth, diff --git a/ogn/collect/takeoff_landings.py b/ogn_python/collect/takeoff_landings.py similarity index 96% rename from ogn/collect/takeoff_landings.py rename to ogn_python/collect/takeoff_landings.py index fdae002..01fa6d8 100644 --- a/ogn/collect/takeoff_landings.py +++ b/ogn_python/collect/takeoff_landings.py @@ -6,9 +6,9 @@ from sqlalchemy import and_, or_, insert, between, exists from sqlalchemy.sql import func, null from sqlalchemy.sql.expression import case -from ogn.collect.celery import app -from ogn.model import AircraftBeacon, TakeoffLanding, Airport -from ogn.utils import date_to_timestamps +from ogn_python.collect.celery import app +from ogn_python.model import AircraftBeacon, TakeoffLanding, Airport +from ogn_python.utils import date_to_timestamps logger = get_task_logger(__name__) @@ -38,7 +38,7 @@ def update_takeoff_landings(session=None, date=None): # limit time range to given date if date is not None: (start, end) = date_to_timestamps(date) - filters = [between(TakeoffLanding.timestamp, start, end)] + filters = [between(AircraftBeacon.timestamp, start, end)] else: filters = [] diff --git a/ogn_python/commands/__init__.py b/ogn_python/commands/__init__.py new file mode 100644 index 0000000..469306a --- /dev/null +++ b/ogn_python/commands/__init__.py @@ -0,0 +1,15 @@ +from ogn_python import app + +from .database import user_cli as database_cli +from .export import user_cli as export_cli +from .flights import user_cli as flights_cli +from .gateway import user_cli as gateway_cli +from .logbook import user_cli as logbook_cli +from .stats import user_cli as stats_cli + +app.cli.add_command(database_cli) +app.cli.add_command(export_cli) +app.cli.add_command(flights_cli) +app.cli.add_command(gateway_cli) +app.cli.add_command(logbook_cli) +app.cli.add_command(stats_cli) diff --git a/ogn/commands/database.py b/ogn_python/commands/database.py similarity index 53% rename from ogn/commands/database.py rename to ogn_python/commands/database.py index b14bd6e..0254711 100644 --- a/ogn/commands/database.py +++ b/ogn_python/commands/database.py @@ -1,13 +1,17 @@ -from datetime import datetime, timedelta +from flask.cli import AppGroup +import click -from manager import Manager -from ogn.collect.database import update_device_infos, update_country_code -from ogn.commands.dbutils import engine, session -from ogn.model import Base, DeviceInfoOrigin, AircraftBeacon -from ogn.utils import get_airports, get_days +from datetime import datetime, timedelta from sqlalchemy.sql import func -manager = Manager() +from ogn_python.collect.database import update_device_infos, update_country_code +from ogn_python.model import * +from ogn_python.utils import get_airports, get_days +from ogn_python import db + +user_cli = AppGroup('database') +user_cli.help = "Database creation and handling." + ALEMBIC_CONFIG_FILE = "alembic.ini" @@ -16,7 +20,7 @@ def get_database_days(start, end): """Returns the first and the last day in aircraft_beacons table.""" if start is None and end is None: - days_from_db = session.query(func.min(AircraftBeacon.timestamp).label('first_day'), func.max(AircraftBeacon.timestamp).label('last_day')).one() + days_from_db = db.session.query(func.min(AircraftBeacon.timestamp).label('first_day'), func.max(AircraftBeacon.timestamp).label('last_day')).one() start = days_from_db[0].date() end = days_from_db[1].date() else: @@ -28,34 +32,43 @@ def get_database_days(start, end): return days -@manager.command +@user_cli.command('info') +def info(): + import importlib + import os + config = importlib.import_module(os.environ['OGN_CONFIG_MODULE']) + print(config) + print(config.SQLALCHEMY_DATABASE_URI) + + +@user_cli.command('init') def init(): """Initialize the database.""" from alembic.config import Config from alembic import command - session.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - session.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;') - session.commit() - Base.metadata.create_all(engine) + db.session.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + db.session.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;') + db.session.commit() + db.create_all() #alembic_cfg = Config(ALEMBIC_CONFIG_FILE) #command.stamp(alembic_cfg, "head") print("Done.") -@manager.command +@user_cli.command('init_timescaledb') def init_timescaledb(): """Initialize TimescaleDB features.""" - session.execute('CREATE EXTENSION IF NOT EXISTS timescaledb;') - session.execute("SELECT create_hypertable('aircraft_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);") - session.execute("SELECT create_hypertable('receiver_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);") - session.commit() + db.session.execute('CREATE EXTENSION IF NOT EXISTS timescaledb;') + db.session.execute("SELECT create_hypertable('aircraft_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);") + db.session.execute("SELECT create_hypertable('receiver_beacons', 'timestamp', chunk_target_size => '2GB', if_not_exists => TRUE);") + db.session.commit() -@manager.command +@user_cli.command('upgrade') def upgrade(): """Upgrade database to the latest version.""" @@ -66,62 +79,66 @@ def upgrade(): command.upgrade(alembic_cfg, 'head') -@manager.command -def drop(sure='n'): +@user_cli.command('drop') +@click.option('--sure', default='n') +def drop(sure): """Drop all tables.""" if sure == 'y': - Base.metadata.drop_all(engine) + db.drop_all() print('Dropped all tables.') else: print("Add argument '--sure y' to drop all tables.") -@manager.command +@user_cli.command('import_ddb') def import_ddb(): """Import registered devices from the DDB.""" print("Import registered devices fom the DDB...") - counter = update_device_infos(session, DeviceInfoOrigin.ogn_ddb) + counter = update_device_infos(db.session, DeviceInfoOrigin.ogn_ddb) print("Imported %i devices." % counter) -@manager.command +@user_cli.command('import_file') +@click.argument('path') def import_file(path='tests/custom_ddb.txt'): """Import registered devices from a local file.""" print("Import registered devices from '{}'...".format(path)) - counter = update_device_infos(session, + counter = update_device_infos(db.session, DeviceInfoOrigin.user_defined, path=path) print("Imported %i devices." % counter) -@manager.command +@user_cli.command('import_flarmnet') +@click.argument('path') def import_flarmnet(path=None): """Import registered devices from a local file.""" print("Import registered devices from '{}'...".format("internet" if path is None else path)) - counter = update_device_infos(session, + counter = update_device_infos(db.session, DeviceInfoOrigin.flarmnet, path=path) print("Imported %i devices." % counter) -@manager.command +@user_cli.command('import_airports') +@click.argument('path') def import_airports(path='tests/SeeYou.cup'): """Import airports from a ".cup" file""" print("Import airports from '{}'...".format(path)) airports = get_airports(path) - session.bulk_save_objects(airports) - session.commit() - session.execute("UPDATE airports SET border = ST_Expand(location, 0.05)") - session.commit() + db.session.bulk_save_objects(airports) + db.session.commit() + db.session.execute("UPDATE airports SET border = ST_Expand(location, 0.05)") + db.session.commit() print("Imported {} airports.".format(len(airports))) -@manager.command +@user_cli.command('update_country_codes') def update_country_codes(): """Update country codes of all receivers.""" - update_country_code(session=session) + update_country_code(session=db.session) diff --git a/ogn/commands/dbutils.py b/ogn_python/commands/dbutils.py similarity index 100% rename from ogn/commands/dbutils.py rename to ogn_python/commands/dbutils.py diff --git a/ogn/commands/export.py b/ogn_python/commands/export.py similarity index 89% rename from ogn/commands/export.py rename to ogn_python/commands/export.py index daec62f..6ce476c 100644 --- a/ogn/commands/export.py +++ b/ogn_python/commands/export.py @@ -1,17 +1,19 @@ +from flask.cli import AppGroup +import click + import datetime import re import csv from aerofiles.igc import Writer -from manager import Manager -from ogn.commands.dbutils import session -from ogn.model import AircraftBeacon, Device +from ogn_python.model import * +from ogn_python import db + +user_cli = AppGroup('export') +user_cli.help = "Export data in several file formats." -manager = Manager() - - -@manager.command +@user_cli.command('cup') def cup(): """Export receiver waypoints as '.cup'.""" @@ -45,17 +47,16 @@ def cup(): INNER JOIN countries c ON c.gid = sq.country_id ORDER BY sq.name; """ - results = session.execute(sql) + results = db.session.execute(sql) with open('receivers.cup', 'w') as outfile: outcsv = csv.writer(outfile) outcsv.writerow(results.keys()) outcsv.writerows(results.fetchall()) - -@manager.arg('address', help='address (flarm id)') -@manager.arg('date', help='date (format: yyyy-mm-dd)') -@manager.command +@user_cli.command('igc') +@click.argument('address') +@click.argument('date') def igc(address, date): """Export igc file for
at .""" if not re.match('.{6}', address): @@ -66,7 +67,7 @@ def igc(address, date): print("Date {} not valid.".format(date)) return - device_id = session.query(Device.id) \ + device_id = db.session.query(Device.id) \ .filter(Device.address == address) \ .first() @@ -95,7 +96,7 @@ def igc(address, date): 'competition_class': 'Doubleseater', }) - points = session.query(AircraftBeacon) \ + points = db.session.query(AircraftBeacon) \ .filter(AircraftBeacon.device_id == device_id) \ .filter(AircraftBeacon.timestamp > date + ' 00:00:00') \ .filter(AircraftBeacon.timestamp < date + ' 23:59:59') \ diff --git a/ogn_python/commands/flights.py b/ogn_python/commands/flights.py new file mode 100644 index 0000000..c963297 --- /dev/null +++ b/ogn_python/commands/flights.py @@ -0,0 +1,136 @@ +from flask.cli import AppGroup +import click + +from datetime import datetime +from tqdm import tqdm + +from ogn_python.commands.database import get_database_days +from ogn_python import db + +user_cli = AppGroup('flights') +user_cli.help = "Create 2D flight paths from data." + +NOTHING = '' +CONTEST_RELEVANT = 'AND agl < 1000' +LOW_PASS = 'AND agl < 50 and ground_speed > 250' + + +def compute_gaps(session, date): + query = """ + INSERT INTO flights2d(date, flight_type, device_id, path) + SELECT '{date}' AS date, + 3 AS flight_type, + sq3.device_id, + ST_Collect(sq3.path) + FROM ( + SELECT sq2.d1 device_id, + ST_MakeLine(sq2.l1, sq2.l2) path + FROM + ( + SELECT sq.timestamp t1, + LAG(sq.timestamp) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) t2, + sq.location l1, + LAG(sq.location) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) l2, + sq.device_id d1, + LAG(sq.device_id) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) d2 + FROM + ( + SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location, agl + FROM aircraft_beacons + WHERE timestamp BETWEEN '{date} 00:00:00' AND '{date} 23:59:59' AND agl > 300 + ORDER BY device_id, timestamp, error_count + ) sq + ) sq2 + WHERE EXTRACT(epoch FROM sq2.t1 - sq2.t2) > 300 + AND ST_DistanceSphere(sq2.l1, sq2.l2) / EXTRACT(epoch FROM sq2.t1 - sq2.t2) BETWEEN 15 AND 50 + ) sq3 + GROUP BY sq3.device_id + ON CONFLICT DO NOTHING; + """.format(date=date.strftime('%Y-%m-%d')) + + session.execute(query) + session.commit() + + +def compute_flights2d(session, date, flight_type): + if flight_type == 0: + filter = NOTHING + elif flight_type == 1: + filter = CONTEST_RELEVANT + elif flight_type == 2: + filter = LOW_PASS + + query = """ + INSERT INTO flights2d + ( + date, + flight_type, + device_id, + path, + path_simple + ) + SELECT '{date}' AS date, + {flight_type} as flight_type, + sq5.device_id, + st_collect(sq5.linestring order BY sq5.part) multilinestring, + st_collect(st_simplify(sq5.linestring, 0.0001) ORDER BY sq5.part) simple_multilinestring + FROM ( + SELECT sq4.device_id, + sq4.part, + st_makeline(sq4.location ORDER BY sq4.timestamp) linestring + FROM ( + SELECT sq3.timestamp, + sq3.location, + sq3.device_id, + sum(sq3.ping) OVER (partition BY sq3.device_id ORDER BY sq3.timestamp) part + FROM ( + SELECT sq2.t1 AS timestamp, + sq2.l1 AS location, + sq2.d1 device_id, + CASE + WHEN sq2.t1 - sq2.t2 < interval'100s' AND ST_DistanceSphere(sq2.l1, sq2.l2) < 1000 THEN 0 + ELSE 1 + END AS ping + FROM ( + SELECT sq.timestamp t1, + lag(sq.timestamp) OVER (partition BY sq.device_id ORDER BY sq.timestamp) t2, + sq.location l1, + lag(sq.location) OVER (partition BY sq.device_id ORDER BY sq.timestamp) l2, + sq.device_id d1, + lag(sq.device_id) OVER (partition BY sq.device_id ORDER BY sq.timestamp) d2 + FROM ( + SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location + FROM aircraft_beacons + WHERE timestamp BETWEEN '{date} 00:00:00' AND '{date} 23:59:59' {filter} + ORDER BY device_id, timestamp, error_count + ) sq + ) sq2 + ) sq3 + ) sq4 + GROUP BY sq4.device_id, sq4.part + ) sq5 + GROUP BY sq5.device_id + ON CONFLICT DO NOTHING; + """.format(date=date.strftime('%Y-%m-%d'), + flight_type=flight_type, + filter=filter) + session.execute(query) + session.commit() + + +@user_cli.command('create') +@click.argument('start') +@click.argument('end') +@click.argument('flight_type', type=click.INT) +def create(start, end, flight_type): + """Compute flights. Flight type: 0: all flights, 1: below 1000m AGL, 2: below 50m AGL + faster than 250 km/h, 3: inverse coverage'""" + + days = get_database_days(start, end) + + pbar = tqdm(days) + for single_date in pbar: + pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) + if flight_type <= 2: + result = compute_flights2d(session=db.session, date=single_date, flight_type=flight_type) + else: + result = compute_gaps(session=db.session, date=single_date) diff --git a/ogn_python/commands/gateway.py b/ogn_python/commands/gateway.py new file mode 100644 index 0000000..2d1039c --- /dev/null +++ b/ogn_python/commands/gateway.py @@ -0,0 +1,34 @@ +from flask.cli import AppGroup +import click + +from ogn.client import AprsClient +from ogn_python.gateway.bulkimport import ContinuousDbFeeder + +from ogn_python import app + +user_cli = AppGroup('gateway') +user_cli.help = "Connection to APRS servers." + + +@user_cli.command('run') +def run(aprs_user='anon-dev'): + """Run the aprs client.""" + + saver = ContinuousDbFeeder() + + # User input validation + if len(aprs_user) < 3 or len(aprs_user) > 9: + print('aprs_user must be a string of 3-9 characters.') + return + + app.logger.warning('Start ogn gateway') + client = AprsClient(aprs_user) + client.connect() + + try: + client.run(callback=saver.add, autoreconnect=True) + except KeyboardInterrupt: + app.logger.warning('\nStop ogn gateway') + + saver.flush() + client.disconnect() diff --git a/ogn/commands/logbook.py b/ogn_python/commands/logbook.py similarity index 77% rename from ogn/commands/logbook.py rename to ogn_python/commands/logbook.py index 164631f..7f07cbb 100644 --- a/ogn/commands/logbook.py +++ b/ogn_python/commands/logbook.py @@ -1,23 +1,27 @@ -# -*- coding: utf-8 -*- +from flask.cli import AppGroup +import click from datetime import datetime -from manager import Manager -from ogn.collect.logbook import update_logbook -from ogn.collect.takeoff_landings import update_takeoff_landings -from ogn.commands.dbutils import session -from ogn.model import Airport, Logbook -from sqlalchemy import or_, between +from ogn_python.collect.logbook import update_logbook +from ogn_python.collect.takeoff_landings import update_takeoff_landings +from ogn_python.commands.dbutils import session +from ogn_python.model import Airport, Logbook from sqlalchemy.sql import func from tqdm import tqdm -from ogn.commands.database import get_database_days -from ogn.utils import date_to_timestamps +from ogn_python.commands.database import get_database_days +from ogn_python.utils import date_to_timestamps -manager = Manager() +from ogn_python import db + +user_cli = AppGroup('logbook') +user_cli.help = "Handling of logbook data." -@manager.command -def compute_takeoff_landing(start=None, end=None): +@user_cli.command('compute_takeoff_landing') +@click.argument('start') +@click.argument('end') +def compute_takeoff_landing(start, end): """Compute takeoffs and landings.""" days = get_database_days(start, end) @@ -28,8 +32,10 @@ def compute_takeoff_landing(start=None, end=None): result = update_takeoff_landings(session=session, date=single_date) -@manager.command -def compute_logbook(start=None, end=None): +@user_cli.command('compute_logbook') +@click.argument('start') +@click.argument('end') +def compute_logbook(start, end): """Compute logbook.""" days = get_database_days(start, end) @@ -40,8 +46,9 @@ def compute_logbook(start=None, end=None): result = update_logbook(session=session, date=single_date) -@manager.arg('date', help='date (format: yyyy-mm-dd)') -@manager.command +@user_cli.command('show') +@click.argument('airport_name') +@click.argument('date') def show(airport_name, date=None): """Show a logbook for .""" airport = session.query(Airport) \ @@ -56,14 +63,14 @@ def show(airport_name, date=None): if date is not None: date = datetime.strptime(date, "%Y-%m-%d") (start, end) = date_to_timestamps(date) - or_args = [between(Logbook.reftime, start, end)] + or_args = [db.between(Logbook.reftime, start, end)] # get all logbook entries and add device and airport infos logbook_query = session.query(func.row_number().over(order_by=Logbook.reftime).label('row_number'), Logbook) \ .filter(*or_args) \ - .filter(or_(Logbook.takeoff_airport_id == airport.id, - Logbook.landing_airport_id == airport.id)) \ + .filter(db.or_(Logbook.takeoff_airport_id == airport.id, + Logbook.landing_airport_id == airport.id)) \ .order_by(Logbook.reftime) # ... and finally print out the logbook diff --git a/ogn_python/commands/stats.py b/ogn_python/commands/stats.py new file mode 100644 index 0000000..5e7d0b1 --- /dev/null +++ b/ogn_python/commands/stats.py @@ -0,0 +1,115 @@ +from flask.cli import AppGroup +import click + +from datetime import datetime +from tqdm import tqdm + +from ogn_python.commands.database import get_database_days + +from ogn_python.collect.stats import create_device_stats, create_receiver_stats, create_relation_stats, create_country_stats,\ + update_qualities, update_receivers as update_receivers_command, update_devices as update_devices_command,\ + update_device_stats_jumps + +from ogn_python.collect.ognrange import create_receiver_coverage + +from ogn_python import db + + +user_cli = AppGroup('stats') +user_cli.help = "Handling of statistical data." + + +@user_cli.command('create') +@click.argument('start') +@click.argument('end') +def create(start, end): + """Create DeviceStats, ReceiverStats and RelationStats.""" + + days = get_database_days(start, end) + + pbar = tqdm(days) + for single_date in pbar: + pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) + result = create_device_stats(session=db.session, date=single_date) + result = update_device_stats_jumps(session=db.session, date=single_date) + result = create_receiver_stats(session=db.session, date=single_date) + result = create_relation_stats(session=db.session, date=single_date) + result = update_qualities(session=db.session, date=single_date) + +@user_cli.command('create_country') +@click.argument('start') +@click.argument('end') +def create_country(start, end): + """Create CountryStats.""" + + days = get_database_days(start, end) + + pbar = tqdm(days) + for single_date in pbar: + pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) + result = create_country_stats(session=db.session, date=single_date) + +from ogn_python.model import * +@user_cli.command('update_devices_name') +def update_devices_name(): + """Update Devices name.""" + + device_ids = db.session.query(Device.id).all() + + for device_id in tqdm(device_ids): + db.session.execute("update devices d set name = sq.name from ( select * from aircraft_beacons ab where ab.device_id = {} limit 1) sq where d.id = sq.device_id and d.name is null or d.name = 'ICA3D3CC4';".format(device_id[0])) + db.session.commit() + + +@user_cli.command('update_receivers') +def update_receivers(): + """Update receivers with data from stats.""" + + result = update_receivers_command(session=db.session) + print(result) + + +@user_cli.command('update_devices') +def update_devices(): + """Update devices with data from stats.""" + + result = update_devices_command(session=db.session) + print(result) + + +@user_cli.command('update_mgrs') +@click.argument('start') +@click.argument('end') +def update_mgrs(start, end): + """Create location_mgrs_short.""" + + days = get_database_days(start, end) + + pbar = tqdm(days) + for single_date in pbar: + datestr = datetime.strftime(single_date, '%Y-%m-%d') + pbar.set_description(datestr) + for pbar2 in tqdm(["{:02d}:{:02d}".format(hh, mm) for hh in range(0, 24) for mm in range(0, 60)]): + sql = """ + UPDATE aircraft_beacons + SET location_mgrs_short = left(location_mgrs, 5) || substring(location_mgrs, 6, 2) || substring(location_mgrs, 11, 2) + WHERE timestamp BETWEEN '{0} {1}:00' and '{0} {1}:59' AND location_mgrs_short IS NULL; + """.format(datestr, pbar2) + + #print(sql) + db.session.execute(sql) + db.session.commit() + + +@user_cli.command('create_ognrange') +@click.argument('start') +@click.argument('end') +def create_ognrange(start=None, end=None): + """Create stats for Melissas ognrange.""" + + days = get_database_days(start, end) + + pbar = tqdm(days) + for single_date in pbar: + pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d')) + result = create_receiver_coverage(session=db.session, date=single_date) diff --git a/ogn/gateway/__init__.py b/ogn_python/gateway/__init__.py similarity index 100% rename from ogn/gateway/__init__.py rename to ogn_python/gateway/__init__.py diff --git a/ogn_python/gateway/bulkimport.py b/ogn_python/gateway/bulkimport.py new file mode 100644 index 0000000..e36facb --- /dev/null +++ b/ogn_python/gateway/bulkimport.py @@ -0,0 +1,322 @@ +from datetime import datetime, timedelta +from io import StringIO + +from flask.cli import AppGroup +import click +from tqdm import tqdm +from mgrs import MGRS + +from ogn.parser import parse, ParseError + +from ogn_python.model import AircraftBeacon, ReceiverBeacon, Location +from ogn_python.utils import open_file +from ogn_python.gateway.process_tools import * + +from ogn_python import db +from ogn_python import app + +user_cli = AppGroup('bulkimport') +user_cli.help = "Tools for accelerated data import." + + +# define message types we want to proceed +AIRCRAFT_BEACON_TYPES = ['aprs_aircraft', 'flarm', 'tracker', 'fanet', 'lt24', 'naviter', 'skylines', 'spider', 'spot'] +RECEIVER_BEACON_TYPES = ['aprs_receiver', 'receiver'] + +# define fields we want to proceed +BEACON_KEY_FIELDS = ['name', 'receiver_name', 'timestamp'] +AIRCRAFT_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'track', 'ground_speed', 'address_type', 'aircraft_type', 'stealth', 'address', 'climb_rate', 'turn_rate', 'signal_quality', 'error_count', 'frequency_offset', 'gps_quality_horizontal', 'gps_quality_vertical', 'software_version', 'hardware_version', 'real_address', 'signal_power', 'distance', 'radial', 'quality', 'location_mgrs', 'location_mgrs_short', 'agl', 'receiver_id', 'device_id'] +RECEIVER_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'version', 'platform', 'cpu_load', 'free_ram', 'total_ram', 'ntp_error', 'rt_crystal_correction', 'voltage', 'amperage', 'cpu_temp', 'senders_visible', 'senders_total', 'rec_input_noise', 'senders_signal', 'senders_messages', 'good_senders_signal', 'good_senders', 'good_and_bad_senders'] + + +myMGRS = MGRS() + + +def string_to_message(raw_string, reference_date): + global receivers + + try: + message = parse(raw_string, reference_date) + except NotImplementedError as e: + app.logger.error('No parser implemented for message: {}'.format(raw_string)) + return None + except ParseError as e: + app.logger.error('Parsing error with message: {}'.format(raw_string)) + return None + except TypeError as e: + app.logger.error('TypeError with message: {}'.format(raw_string)) + return None + except Exception as e: + app.logger.error('Other Exception with string: {}'.format(raw_string)) + return None + + # update reference receivers and distance to the receiver + if message['aprs_type'] == 'position': + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES + RECEIVER_BEACON_TYPES: + latitude = message['latitude'] + longitude = message['longitude'] + + location = Location(longitude, latitude) + message['location'] = location.to_wkt() + location_mgrs = myMGRS.toMGRS(latitude, longitude).decode('utf-8') + message['location_mgrs'] = location_mgrs + message['location_mgrs_short'] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] + + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and 'gps_quality' in message: + if message['gps_quality'] is not None and 'horizontal' in message['gps_quality']: + message['gps_quality_horizontal'] = message['gps_quality']['horizontal'] + message['gps_quality_vertical'] = message['gps_quality']['vertical'] + del message['gps_quality'] + + # TODO: Fix python-ogn-client 0.91 + if 'senders_messages' in message and message['senders_messages'] is not None: + message['senders_messages'] = int(message['senders_messages']) + if 'good_senders' in message and message['good_senders'] is not None: + message['good_senders'] = int(message['good_senders']) + if 'good_and_bad_senders' in message and message['good_and_bad_senders'] is not None: + message['good_and_bad_senders'] = int(message['good_and_bad_senders']) + + return message + + +class ContinuousDbFeeder: + def __init__(self,): + self.postfix = 'continuous_import' + self.last_flush = datetime.utcnow() + self.last_add_missing = datetime.utcnow() + self.last_transfer = datetime.utcnow() + + self.aircraft_buffer = StringIO() + self.receiver_buffer = StringIO() + + create_tables(self.postfix) + create_indices(self.postfix) + + def add(self, raw_string): + message = string_to_message(raw_string, reference_date=datetime.utcnow()) + + if message is None or ('raw_message' in message and message['raw_message'][0] == '#') or 'beacon_type' not in message: + return + + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: + complete_message = ','.join([str(message[k]) if k in message and message[k] is not None else '\\N' for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS]) + self.aircraft_buffer.write(complete_message) + self.aircraft_buffer.write('\n') + elif message['beacon_type'] in RECEIVER_BEACON_TYPES: + complete_message = ','.join([str(message[k]) if k in message and message[k] is not None else '\\N' for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS]) + self.receiver_buffer.write(complete_message) + self.receiver_buffer.write('\n') + else: + app.logger.error("Ignore beacon_type: {}".format(message['beacon_type'])) + return + + if datetime.utcnow() - self.last_flush >= timedelta(seconds=5): + self.flush() + self.prepare() + + self.aircraft_buffer = StringIO() + self.receiver_buffer = StringIO() + + self.last_flush = datetime.utcnow() + + if datetime.utcnow() - self.last_add_missing >= timedelta(seconds=60): + self.add_missing() + self.last_add_missing = datetime.utcnow() + + if datetime.utcnow() - self.last_transfer >= timedelta(seconds=10): + self.transfer() + self.delete_beacons() + self.last_transfer = datetime.utcnow() + + + def flush(self): + self.aircraft_buffer.seek(0) + self.receiver_buffer.seek(0) + + connection = db.engine.raw_connection() + cursor = connection.cursor() + cursor.copy_from(self.aircraft_buffer, 'aircraft_beacons_{0}'.format(self.postfix), sep=',', columns=BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS) + cursor.copy_from(self.receiver_buffer, 'receiver_beacons_{0}'.format(self.postfix), sep=',', columns=BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS) + connection.commit() + + self.aircraft_buffer = StringIO() + self.receiver_buffer = StringIO() + + def add_missing(self): + add_missing_receivers(self.postfix) + add_missing_devices(self.postfix) + + def prepare(self): + # make receivers complete + update_receiver_beacons(self.postfix) + update_receiver_location(self.postfix) + + # make devices complete + update_aircraft_beacons(self.postfix) + + def transfer(self): + # tranfer beacons + transfer_aircraft_beacons(self.postfix) + transfer_receiver_beacons(self.postfix) + + def delete_beacons(self): + # delete already transfered beacons + delete_receiver_beacons(self.postfix) + delete_aircraft_beacons(self.postfix) + + +class FileDbFeeder(): + def __init__(self): + self.postfix = 'continuous_import' + self.last_flush = datetime.utcnow() + + self.aircraft_buffer = StringIO() + self.receiver_buffer = StringIO() + + create_tables(self.postfix) + create_indices(self.postfix) + + def add(self, raw_string): + message = string_to_message(raw_string, reference_date=datetime.utcnow()) + + if message is None or ('raw_message' in message and message['raw_message'][0] == '#') or 'beacon_type' not in message: + return + + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: + complete_message = ','.join([str(message[k]) if k in message and message[k] is not None else '\\N' for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS]) + self.aircraft_buffer.write(complete_message) + self.aircraft_buffer.write('\n') + elif message['beacon_type'] in RECEIVER_BEACON_TYPES: + complete_message = ','.join([str(message[k]) if k in message and message[k] is not None else '\\N' for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS]) + self.receiver_buffer.write(complete_message) + self.receiver_buffer.write('\n') + else: + app.logger.error("Ignore beacon_type: {}".format(message['beacon_type'])) + return + + def prepare(self): + # make receivers complete + add_missing_receivers(self.postfix) + update_receiver_location(self.postfix) + + # make devices complete + add_missing_devices(self.postfix) + + # prepare beacons for transfer + create_indices(self.postfix) + update_receiver_beacons_bigdata(self.postfix) + update_aircraft_beacons_bigdata(self.postfix) + + +def get_aircraft_beacons_postfixes(): + """Get the postfixes from imported aircraft_beacons logs.""" + + postfixes = db.session.execute(""" + SELECT DISTINCT(RIGHT(tablename, 8)) + FROM pg_catalog.pg_tables + WHERE schemaname = 'public' AND tablename LIKE 'aircraft\_beacons\_20______' + ORDER BY RIGHT(tablename, 10); + """).fetchall() + + return [postfix for postfix in postfixes] + + + + + +def export_to_path(postfix): + import os, gzip + aircraft_beacons_file = os.path.join(path, 'aircraft_beacons_{0}.csv.gz'.format(postfix)) + with gzip.open(aircraft_beacons_file, 'wt', encoding='utf-8') as gzip_file: + self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_aircraft_beacons_subquery()), gzip_file) + receiver_beacons_file = os.path.join(path, 'receiver_beacons_{0}.csv.gz'.format(postfix)) + with gzip.open(receiver_beacons_file, 'wt') as gzip_file: + self.cur.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format(self.get_merged_receiver_beacons_subquery()), gzip_file) + + + +def convert(sourcefile, datestr, saver): + from ogn_python.gateway.process import string_to_message + from ogn_python.gateway.process_tools import AIRCRAFT_BEACON_TYPES, RECEIVER_BEACON_TYPES + from datetime import datetime + + fin = open_file(sourcefile) + + # get total lines of the input file + total_lines = 0 + for line in fin: + total_lines += 1 + fin.seek(0) + + current_line = 0 + steps = 100000 + reference_date = datetime.strptime(datestr + ' 12:00:00', '%Y-%m-%d %H:%M:%S') + + pbar = tqdm(fin, total=total_lines) + for line in pbar: + pbar.set_description('Importing {}'.format(sourcefile)) + + current_line += 1 + if current_line % steps == 0: + saver.flush() + + message = string_to_message(line.strip(), reference_date=reference_date) + if message is None: + continue + + dictfilt = lambda x, y: dict([(i, x[i]) for i in x if i in set(y)]) + + try: + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: + message = dictfilt(message, ('beacon_type', 'aprs_type', 'location_wkt', 'altitude', 'name', 'dstcall', 'relay', 'receiver_name', 'timestamp', 'track', 'ground_speed', + 'address_type', 'aircraft_type', 'stealth', 'address', 'climb_rate', 'turn_rate', 'signal_quality', 'error_count', 'frequency_offset', 'gps_quality_horizontal', 'gps_quality_vertical', 'software_version', 'hardware_version', 'real_address', 'signal_power', + 'distance', 'radial', 'quality', 'agl', 'location_mgrs', 'location_mgrs_short', + 'receiver_id', 'device_id')) + + beacon = AircraftBeacon(**message) + elif message['beacon_type'] in RECEIVER_BEACON_TYPES: + if 'rec_crystal_correction' in message: + del message['rec_crystal_correction'] + del message['rec_crystal_correction_fine'] + beacon = ReceiverBeacon(**message) + saver.add(beacon) + except Exception as e: + print(e) + + saver.flush() + fin.close() + + +@user_cli.command('file_import') +@click.argument('path') +def file_import(path): + """Import APRS logfiles into separate logfile tables.""" + + import os + import re + + # Get Filepaths and dates to import + results = list() + for (root, dirs, files) in os.walk(path): + for file in sorted(files): + match = re.match('OGN_log\.txt_([0-9]{4}\-[0-9]{2}\-[0-9]{2})\.gz$', file) + if match: + results.append({'filepath': os.path.join(root, file), + 'datestr': match.group(1)}) + + with LogfileDbSaver() as saver: + already_imported = saver.get_datestrs() + + results = list(filter(lambda x: x['datestr'] not in already_imported, results)) + + pbar = tqdm(results) + for result in pbar: + filepath = result['filepath'] + datestr = result['datestr'] + pbar.set_description("Importing data for {}".format(datestr)) + + saver.set_datestr(datestr) + saver.create_tables() + convert(filepath, datestr, saver) + saver.add_missing_devices() + saver.add_missing_receivers() diff --git a/ogn_python/gateway/process_tools.py b/ogn_python/gateway/process_tools.py new file mode 100644 index 0000000..2b178b8 --- /dev/null +++ b/ogn_python/gateway/process_tools.py @@ -0,0 +1,322 @@ +from ogn_python import db + +def create_tables(postfix): + """Create tables for log file import.""" + + db.session.execute('DROP TABLE IF EXISTS "aircraft_beacons_{0}"; CREATE TABLE aircraft_beacons_{0} AS TABLE aircraft_beacons WITH NO DATA;'.format(postfix)) + db.session.execute('DROP TABLE IF EXISTS "receiver_beacons_{0}"; CREATE TABLE receiver_beacons_{0} AS TABLE receiver_beacons WITH NO DATA;'.format(postfix)) + db.session.commit() + + +def create_indices(postfix): + """Creates indices for aircraft- and receiver-beacons.""" + + db.session.execute(""" + CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_device_id ON "aircraft_beacons_{0}" (device_id NULLS FIRST); + CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_receiver_id ON "aircraft_beacons_{0}" (receiver_id NULLS FIRST); + CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_timestamp_name_receiver_name ON "aircraft_beacons_{0}" (timestamp, name, receiver_name); + CREATE INDEX IF NOT EXISTS ix_receiver_beacons_{0}_timestamp_name_receiver_name ON "receiver_beacons_{0}" (timestamp, name, receiver_name); + """.format(postfix)) + db.session.commit() + + +def create_indices_bigdata(postfix): + """Creates indices for aircraft- and receiver-beacons.""" + + db.session.execute(""" + CREATE INDEX IF NOT EXISTS ix_aircraft_beacons_{0}_timestamp_name_receiver_name ON "aircraft_beacons_{0}" (timestamp, name, receiver_name); + CREATE INDEX IF NOT EXISTS ix_receiver_beacons_{0}_timestamp_name_receiver_name ON "receiver_beacons_{0}" (timestamp, name, receiver_name); + """.format(postfix)) + db.session.commit() + + +def add_missing_devices(postfix): + """Add missing devices.""" + + db.session.execute(""" + INSERT INTO devices(address) + SELECT DISTINCT (ab.address) + FROM "aircraft_beacons_{0}" AS ab + WHERE ab.address IS NOT NULL AND NOT EXISTS (SELECT 1 FROM devices AS d WHERE d.address = ab.address) + ORDER BY ab.address; + """.format(postfix)) + db.session.commit() + + +def add_missing_receivers(postfix): + """Add missing receivers.""" + + db.session.execute(""" + INSERT INTO receivers(name) + SELECT DISTINCT (rb.name) + FROM "receiver_beacons_{0}" AS rb + WHERE NOT EXISTS (SELECT 1 FROM receivers AS r WHERE r.name = rb.name) + ORDER BY rb.name; + + INSERT INTO receivers(name) + SELECT DISTINCT (ab.receiver_name) + FROM "aircraft_beacons_{0}" AS ab + WHERE NOT EXISTS (SELECT 1 FROM receivers AS r WHERE r.name = ab.receiver_name) + ORDER BY ab.receiver_name; + """.format(postfix)) + db.session.commit() + + +def update_receiver_location(postfix): + """Updates the receiver location. We need this because we want the actual location for distance calculations.""" + + db.session.execute(""" + UPDATE receivers AS r + SET + location = sq.location, + altitude = sq.altitude + FROM ( + SELECT DISTINCT ON (rb.receiver_id) rb.receiver_id, rb.location, rb.altitude + FROM "receiver_beacons_{0}" AS rb + WHERE rb.location IS NOT NULL + ORDER BY rb.receiver_id, rb.timestamp + ) AS sq + WHERE r.id = sq.receiver_id; + """.format(postfix)) + db.session.commit() + + +def update_receiver_beacons(postfix): + """Updates the foreign keys.""" + + db.session.execute(""" + UPDATE receiver_beacons_{0} AS rb + SET receiver_id = r.id + FROM receivers AS r + WHERE rb.receiver_id IS NULL AND rb.name = r.name; + """.format(postfix)) + db.session.commit() + + +def update_receiver_beacons_bigdata(postfix): + """Updates the foreign keys. + Due to performance reasons we use a new table instead of updating the old.""" + + db.session.execute(""" + SELECT + rb.location, rb.altitude, rb.name, rb.receiver_name, rb.dstcall, rb.timestamp, + + rb.version, rb.platform, rb.cpu_load, rb.free_ram, rb.total_ram, rb.ntp_error, rb.rt_crystal_correction, rb.voltage, rb.amperage, + rb.cpu_temp, rb.senders_visible, rb.senders_total, rb.rec_input_noise, rb.senders_signal, rb.senders_messages, rb.good_senders_signal, + rb.good_senders, rb.good_and_bad_senders, + + r.id AS receiver_id + INTO "receiver_beacons_{0}_temp" + FROM "receiver_beacons_{0}" AS rb, receivers AS r + WHERE rb.name = r.name; + + DROP TABLE IF EXISTS "receiver_beacons_{0}"; + ALTER TABLE "receiver_beacons_{0}_temp" RENAME TO "receiver_beacons_{0}"; + """.format(postfix)) + db.session.commit() + + +def update_aircraft_beacons(postfix): + """Updates the foreign keys and calculates distance/radial and quality and computes the altitude above ground level. + Elevation data has to be in the table 'elevation' with srid 4326.""" + + db.session.execute(""" + UPDATE aircraft_beacons_{0} AS ab + SET + device_id = d.id, + receiver_id = r.id, + distance = CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END, + radial = CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) ELSE NULL END, + quality = CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL + THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL) + ELSE NULL + END + + FROM devices AS d, receivers AS r + WHERE ab.device_id IS NULL and ab.receiver_id IS NULL AND ab.address = d.address AND ab.receiver_name = r.name; + """.format(postfix)) + db.session.commit() + + +def update_aircraft_beacons_bigdata(postfix): + """Updates the foreign keys and calculates distance/radial and quality and computes the altitude above ground level. + Elevation data has to be in the table 'elevation' with srid 4326. + Due to performance reasons we use a new table instead of updating the old.""" + + db.session.execute(""" + SELECT + ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed, + + ab.address_type, ab.aircraft_type, ab.stealth, ab.address, ab.climb_rate, ab.turn_rate, ab.signal_quality, ab.error_count, + ab.frequency_offset, ab.gps_quality_horizontal, ab.gps_quality_vertical, ab.software_version, ab.hardware_version, ab.real_address, ab.signal_power, + + ab.location_mgrs, + ab.location_mgrs_short, + + d.id AS device_id, + r.id AS receiver_id, + CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END AS distance, + CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) ELSE NULL END AS radial, + CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL + THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL) + ELSE NULL + END AS quality, + CAST(ab.altitude - ST_Value(e.rast, ab.location) AS REAL) AS agl + + INTO "aircraft_beacons_{0}_temp" + FROM "aircraft_beacons_{0}" AS ab, devices AS d, receivers AS r, elevation AS e + WHERE ab.address = d.address AND receiver_name = r.name AND ST_Intersects(e.rast, ab.location); + + DROP TABLE IF EXISTS "aircraft_beacons_{0}"; + ALTER TABLE "aircraft_beacons_{0}_temp" RENAME TO "aircraft_beacons_{0}"; + """.format(postfix)) + db.session.commit() + + +def delete_receiver_beacons(postfix): + """Delete beacons from table.""" + + db.session.execute(""" + DELETE FROM receiver_beacons_continuous_import AS rb + USING ( + SELECT name, receiver_name, timestamp + FROM receiver_beacons_continuous_import + WHERE receiver_id IS NOT NULL + ) AS sq + WHERE rb.name = sq.name AND rb.receiver_name = sq.receiver_name AND rb.timestamp = sq.timestamp + """.format(postfix)) + db.session.commit() + + +def delete_aircraft_beacons(postfix): + """Delete beacons from table.""" + + db.session.execute(""" + DELETE FROM aircraft_beacons_continuous_import AS ab + USING ( + SELECT name, receiver_name, timestamp + FROM aircraft_beacons_continuous_import + WHERE receiver_id IS NOT NULL and device_id IS NOT NULL + ) AS sq + WHERE ab.name = sq.name AND ab.receiver_name = sq.receiver_name AND ab.timestamp = sq.timestamp + """.format(postfix)) + db.session.commit() + + +def get_merged_aircraft_beacons_subquery(postfix): + """Some beacons are split into position and status beacon. With this query we merge them into one beacon.""" + + return """ + SELECT + ST_AsEWKT(MAX(location)) AS location, + MAX(altitude) AS altitude, + name, + MAX(dstcall) AS dstcall, + MAX(relay) AS relay, + receiver_name, + timestamp, + MAX(track) AS track, + MAX(ground_speed) AS ground_speed, + + MAX(address_type) AS address_type, + MAX(aircraft_type) AS aircraft_type, + CAST(MAX(CAST(stealth AS int)) AS boolean) AS stealth, + MAX(address) AS address, + MAX(climb_rate) AS climb_rate, + MAX(turn_rate) AS turn_rate, + MAX(signal_quality) AS signal_quality, + MAX(error_count) AS error_count, + MAX(frequency_offset) AS frequency_offset, + MAX(gps_quality_horizontal) AS gps_quality_horizontal, + MAX(gps_quality_vertical) AS gps_quality_vertical, + MAX(software_version) AS software_version, + MAX(hardware_version) AS hardware_version, + MAX(real_address) AS real_address, + MAX(signal_power) AS signal_power, + + CAST(MAX(distance) AS REAL) AS distance, + CAST(MAX(radial) AS REAL) AS radial, + CAST(MAX(quality) AS REAL) AS quality, + CAST(MAX(agl) AS REAL) AS agl, + MAX(location_mgrs) AS location_mgrs, + MAX(location_mgrs_short) AS location_mgrs_short, + + MAX(receiver_id) AS receiver_id, + MAX(device_id) AS device_id + FROM "aircraft_beacons_{0}" AS ab + GROUP BY timestamp, name, receiver_name + ORDER BY timestamp, name, receiver_name + """.format(postfix) + + +def get_merged_receiver_beacons_subquery(postfix): + """Some beacons are split into position and status beacon. With this query we merge them into one beacon.""" + + return """ + SELECT + ST_AsEWKT(MAX(location)) AS location, + MAX(altitude) AS altitude, + name, + receiver_name, + MAX(dstcall) AS dstcall, + timestamp, + + MAX(version) AS version, + MAX(platform) AS platform, + MAX(cpu_load) AS cpu_load, + MAX(free_ram) AS free_ram, + MAX(total_ram) AS total_ram, + MAX(ntp_error) AS ntp_error, + MAX(rt_crystal_correction) AS rt_crystal_correction, + MAX(voltage) AS voltage, + MAX(amperage) AS amperage, + MAX(cpu_temp) AS cpu_temp, + MAX(senders_visible) AS senders_visible, + MAX(senders_total) AS senders_total, + MAX(rec_input_noise) AS rec_input_noise, + MAX(senders_signal) AS senders_signal, + MAX(senders_messages) AS senders_messages, + MAX(good_senders_signal) AS good_senders_signal, + MAX(good_senders) AS good_senders, + MAX(good_and_bad_senders) AS good_and_bad_senders, + + MAX(receiver_id) AS receiver_id + FROM "receiver_beacons_{0}" AS rb + GROUP BY timestamp, name, receiver_name + ORDER BY timestamp, name, receiver_name + """.format(postfix) + + +def transfer_aircraft_beacons(postfix): + query = """ + INSERT INTO aircraft_beacons(location, altitude, name, dstcall, relay, receiver_name, timestamp, track, ground_speed, + address_type, aircraft_type, stealth, address, climb_rate, turn_rate, signal_quality, error_count, frequency_offset, gps_quality_horizontal, gps_quality_vertical, software_version, hardware_version, real_address, signal_power, + distance, radial, quality, agl, location_mgrs, location_mgrs_short, + receiver_id, device_id) + SELECT sq.* + FROM ({}) sq + WHERE sq.receiver_id IS NOT NULL AND sq.device_id IS NOT NULL + ON CONFLICT DO NOTHING; + """.format(get_merged_aircraft_beacons_subquery(postfix)) + + db.session.execute(query) + db.session.commit() + + +def transfer_receiver_beacons(postfix): + query = """ + INSERT INTO receiver_beacons(location, altitude, name, receiver_name, dstcall, timestamp, + + version, platform, cpu_load, free_ram, total_ram, ntp_error, rt_crystal_correction, voltage, + amperage, cpu_temp, senders_visible, senders_total, rec_input_noise, senders_signal, + senders_messages, good_senders_signal, good_senders, good_and_bad_senders, + + receiver_id) + SELECT sq.* + FROM ({}) sq + WHERE sq.receiver_id IS NOT NULL + ON CONFLICT DO NOTHING; + """.format(get_merged_receiver_beacons_subquery(postfix)) + + db.session.execute(query) + db.session.commit() diff --git a/ogn/model/__init__.py b/ogn_python/model/__init__.py similarity index 94% rename from ogn/model/__init__.py rename to ogn_python/model/__init__.py index f302afc..dcd1818 100644 --- a/ogn/model/__init__.py +++ b/ogn_python/model/__init__.py @@ -1,8 +1,8 @@ # flake8: noqa from .aircraft_type import AircraftType -from .base import Base from .beacon import Beacon from .country import Country +from .country_stats import CountryStats from .device import Device from .device_info import DeviceInfo from .device_info_origin import DeviceInfoOrigin diff --git a/ogn/model/aircraft_beacon.py b/ogn_python/model/aircraft_beacon.py similarity index 60% rename from ogn/model/aircraft_beacon.py rename to ogn_python/model/aircraft_beacon.py index aed1cd8..acf5858 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn_python/model/aircraft_beacon.py @@ -1,48 +1,48 @@ -from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger, ForeignKey, Index -from sqlalchemy.orm import relationship from sqlalchemy.sql import func from .beacon import Beacon +from ogn_python import db + class AircraftBeacon(Beacon): __tablename__ = "aircraft_beacons" # Flarm specific data - address_type = Column(SmallInteger) - aircraft_type = Column(SmallInteger) - stealth = Column(Boolean) - address = Column(String) - climb_rate = Column(Float(precision=2)) - turn_rate = Column(Float(precision=2)) - signal_quality = Column(Float(precision=2)) - error_count = Column(SmallInteger) - frequency_offset = Column(Float(precision=2)) - gps_quality_horizontal = Column(SmallInteger) - gps_quality_vertical = Column(SmallInteger) - software_version = Column(Float(precision=2)) - hardware_version = Column(SmallInteger) - real_address = Column(String(6)) - signal_power = Column(Float(precision=2)) + address_type = db.Column(db.SmallInteger) + aircraft_type = db.Column(db.SmallInteger) + stealth = db.Column(db.Boolean) + address = db.Column(db.String) + climb_rate = db.Column(db.Float(precision=2)) + turn_rate = db.Column(db.Float(precision=2)) + signal_quality = db.Column(db.Float(precision=2)) + error_count = db.Column(db.SmallInteger) + frequency_offset = db.Column(db.Float(precision=2)) + gps_quality_horizontal = db.Column(db.SmallInteger) + gps_quality_vertical = db.Column(db.SmallInteger) + software_version = db.Column(db.Float(precision=2)) + hardware_version = db.Column(db.SmallInteger) + real_address = db.Column(db.String(6)) + signal_power = db.Column(db.Float(precision=2)) proximity = None # Calculated values - distance = Column(Float(precision=2)) - radial = Column(SmallInteger) - quality = Column(Float(precision=2)) # signal quality normalized to 10km - location_mgrs = Column(String(15)) # full mgrs (15 chars) - location_mgrs_short = Column(String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool - agl = Column(Float(precision=2)) + distance = db.Column(db.Float(precision=2)) + radial = db.Column(db.SmallInteger) + quality = db.Column(db.Float(precision=2)) # signal quality normalized to 10km + location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) + location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool + agl = db.Column(db.Float(precision=2)) # Relations - receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL')) - receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='aircraft_beacons') + receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL')) + receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref='aircraft_beacons') - device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL')) - device = relationship('Device', foreign_keys=[device_id], backref='aircraft_beacons') + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL')) + device = db.relationship('Device', foreign_keys=[device_id], backref='aircraft_beacons') # Multi-column indices - Index('ix_aircraft_beacons_receiver_id_distance', 'receiver_id', 'distance') - Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp') + db.Index('ix_aircraft_beacons_receiver_id_distance', 'receiver_id', 'distance') + db.Index('ix_aircraft_beacons_device_id_timestamp', 'device_id', 'timestamp') def __repr__(self): return "" % ( @@ -143,5 +143,5 @@ class AircraftBeacon(Beacon): self.location_mgrs_short] -Index('ix_aircraft_beacons_date_device_id_address', func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address) -Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance) +db.Index('ix_aircraft_beacons_date_device_id_address', func.date(AircraftBeacon.timestamp), AircraftBeacon.device_id, AircraftBeacon.address) +db.Index('ix_aircraft_beacons_date_receiver_id_distance', func.date(AircraftBeacon.timestamp), AircraftBeacon.receiver_id, AircraftBeacon.distance) diff --git a/ogn/model/aircraft_type.py b/ogn_python/model/aircraft_type.py similarity index 100% rename from ogn/model/aircraft_type.py rename to ogn_python/model/aircraft_type.py diff --git a/ogn_python/model/airport.py b/ogn_python/model/airport.py new file mode 100644 index 0000000..7f9504c --- /dev/null +++ b/ogn_python/model/airport.py @@ -0,0 +1,37 @@ +from geoalchemy2.types import Geometry + +from ogn_python import db + + +class Airport(db.Model): + __tablename__ = "airports" + + id = db.Column(db.Integer, primary_key=True) + + location_wkt = db.Column('location', Geometry('POINT', srid=4326)) + altitude = db.Column(db.Float(precision=2)) + + name = db.Column(db.String, index=True) + code = db.Column(db.String(6)) + country_code = db.Column(db.String(2)) + style = db.Column(db.SmallInteger) + description = db.Column(db.String) + runway_direction = db.Column(db.SmallInteger) + runway_length = db.Column(db.SmallInteger) + frequency = db.Column(db.Float(precision=2)) + + border = db.Column('border', Geometry('POLYGON', srid=4326)) + + def __repr__(self): + return "" % ( + self.name, + self.code, + self.country_code, + self.style, + self.description, + self.location_wkt.latitude if self.location_wkt else None, + self.location_wkt.longitude if self.location_wkt else None, + self.altitude, + self.runway_direction, + self.runway_length, + self.frequency) diff --git a/ogn/model/beacon.py b/ogn_python/model/beacon.py similarity index 56% rename from ogn/model/beacon.py rename to ogn_python/model/beacon.py index 13bf891..a394f8c 100644 --- a/ogn/model/beacon.py +++ b/ogn_python/model/beacon.py @@ -1,27 +1,27 @@ from geoalchemy2.shape import to_shape from geoalchemy2.types import Geometry -from sqlalchemy import Column, String, SmallInteger, Float, DateTime from sqlalchemy.ext.declarative import AbstractConcreteBase from sqlalchemy.ext.hybrid import hybrid_property -from .base import Base from .geo import Location +from ogn_python import db -class Beacon(AbstractConcreteBase, Base): + +class Beacon(AbstractConcreteBase, db.Model): # APRS data - location_wkt = Column('location', Geometry('POINT', srid=4326)) - altitude = Column(Float(precision=2)) + location_wkt = db.Column('location', Geometry('POINT', srid=4326)) + altitude = db.Column(db.Float(precision=2)) - name = Column(String, primary_key=True, nullable=True) - dstcall = Column(String) - relay = Column(String) - receiver_name = Column(String(9), primary_key=True, nullable=True) - timestamp = Column(DateTime, primary_key=True, nullable=True) + name = db.Column(db.String, primary_key=True, nullable=True) + dstcall = db.Column(db.String) + relay = db.Column(db.String) + receiver_name = db.Column(db.String(9), primary_key=True, nullable=True) + timestamp = db.Column(db.DateTime, primary_key=True, nullable=True) symboltable = None symbolcode = None - track = Column(SmallInteger) - ground_speed = Column(Float(precision=2)) + track = db.Column(db.SmallInteger) + ground_speed = db.Column(db.Float(precision=2)) comment = None # Type information diff --git a/ogn_python/model/country.py b/ogn_python/model/country.py new file mode 100644 index 0000000..2b16304 --- /dev/null +++ b/ogn_python/model/country.py @@ -0,0 +1,38 @@ +from geoalchemy2.types import Geometry + +from ogn_python import db + + +class Country(db.Model): + __tablename__ = "countries" + + gid = db.Column(db.Integer, primary_key=True) + + fips = db.Column(db.String(2)) + iso2 = db.Column(db.String(2)) + iso3 = db.Column(db.String(3)) + + un = db.Column(db.SmallInteger) + name = db.Column(db.String(50)) + area = db.Column(db.Integer) + pop2005 = db.Column(db.BigInteger) + region = db.Column(db.SmallInteger) + subregion = db.Column(db.SmallInteger) + lon = db.Column(db.Float) + lat = db.Column(db.Float) + + geom = db.Column('geom', Geometry('MULTIPOLYGON', srid=4326)) + + def __repr__(self): + return "" % ( + self.fips, + self.iso2, + self.iso3, + self.un, + self.name, + self.area, + self.pop2005, + self.region, + self.subregion, + self.lon, + self.lat) diff --git a/ogn_python/model/country_stats.py b/ogn_python/model/country_stats.py new file mode 100644 index 0000000..c9e3ece --- /dev/null +++ b/ogn_python/model/country_stats.py @@ -0,0 +1,18 @@ +from ogn_python import db + + +class CountryStats(db.Model): + __tablename__ = "country_stats" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + + # Static data + aircraft_beacon_count = db.Column(db.Integer) + device_count = db.Column(db.Integer) + + # Relations + country_id = db.Column(db.Integer, db.ForeignKey('countries.gid', ondelete='SET NULL'), index=True) + country = db.relationship('Country', foreign_keys=[country_id], backref=db.backref('stats', order_by='CountryStats.date.asc()')) + diff --git a/ogn_python/model/device.py b/ogn_python/model/device.py new file mode 100644 index 0000000..fc7d868 --- /dev/null +++ b/ogn_python/model/device.py @@ -0,0 +1,27 @@ +from ogn_python import db + + +class Device(db.Model): + __tablename__ = 'devices' + + id = db.Column(db.Integer, primary_key=True) + + name = db.Column(db.String, index=True) + #address = db.Column(db.String(6), index=True) + address = db.Column(db.String, index=True) + firstseen = db.Column(db.DateTime, index=True) + lastseen = db.Column(db.DateTime, index=True) + aircraft_type = db.Column(db.SmallInteger, index=True) + stealth = db.Column(db.Boolean) + software_version = db.Column(db.Float(precision=2)) + hardware_version = db.Column(db.SmallInteger) + real_address = db.Column(db.String(6)) + + def __repr__(self): + return "" % ( + self.address, + self.aircraft_type, + self.stealth, + self.software_version, + self.hardware_version, + self.real_address) diff --git a/ogn_python/model/device_info.py b/ogn_python/model/device_info.py new file mode 100644 index 0000000..6060506 --- /dev/null +++ b/ogn_python/model/device_info.py @@ -0,0 +1,34 @@ +from ogn_python import db + + +class DeviceInfo(db.Model): + __tablename__ = 'device_infos' + + id = db.Column(db.Integer, primary_key=True) + address_type = None + #address = db.Column(db.String(6), index=True) + address = db.Column(db.String, index=True) + aircraft = db.Column(db.String) + registration = db.Column(db.String(7)) + competition = db.Column(db.String(3)) + tracked = db.Column(db.Boolean) + identified = db.Column(db.Boolean) + aircraft_type = db.Column(db.SmallInteger) + + address_origin = db.Column(db.SmallInteger) + + # Relations + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), index=True) + device = db.relationship('Device', foreign_keys=[device_id], backref=db.backref('infos', order_by='DeviceInfo.address_origin.asc()')) + + def __repr__(self): + return "" % ( + self.address_type, + self.address, + self.aircraft, + self.registration, + self.competition, + self.tracked, + self.identified, + self.aircraft_type, + self.address_origin) diff --git a/ogn/model/device_info_origin.py b/ogn_python/model/device_info_origin.py similarity index 100% rename from ogn/model/device_info_origin.py rename to ogn_python/model/device_info_origin.py diff --git a/ogn_python/model/device_stats.py b/ogn_python/model/device_stats.py new file mode 100644 index 0000000..8865a7d --- /dev/null +++ b/ogn_python/model/device_stats.py @@ -0,0 +1,54 @@ +from ogn_python import db + + +class DeviceStats(db.Model): + __tablename__ = "device_stats" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + + # Static data + name = db.Column(db.String) + firstseen = db.Column(db.DateTime) + lastseen = db.Column(db.DateTime) + aircraft_type = db.Column(db.SmallInteger) + stealth = db.Column(db.Boolean) + software_version = db.Column(db.Float(precision=2)) + hardware_version = db.Column(db.SmallInteger) + real_address = db.Column(db.String(6)) + + # Statistic data + max_altitude = db.Column(db.Float(precision=2)) + receiver_count = db.Column(db.SmallInteger) + aircraft_beacon_count = db.Column(db.Integer) + jumps = db.Column(db.SmallInteger) + ambiguous = db.Column(db.Boolean) + quality = db.Column(db.Float(precision=2)) + + # Relation statistic data + quality_offset = db.Column(db.Float(precision=2)) + + # Ranking data + max_altitude_ranking_worldwide = db.Column(db.Integer) + max_altitude_ranking_country = db.Column(db.Integer) + receiver_count_ranking_worldwide = db.Column(db.Integer) + receiver_count_ranking_country = db.Column(db.Integer) + aircraft_beacon_count_ranking_worldwide = db.Column(db.Integer) + aircraft_beacon_count_ranking_country = db.Column(db.Integer) + quality_ranking_worldwide = db.Column(db.Integer) + quality_ranking_country = db.Column(db.Integer) + + # Relations + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), index=True) + device = db.relationship('Device', foreign_keys=[device_id], backref=db.backref('stats', order_by='DeviceStats.date.asc()')) + + def __repr__(self): + return "" % ( + self.date, + self.receiver_count, + self.aircraft_beacon_count, + self.max_altitude) + + +db.Index('ix_device_stats_date_device_id', DeviceStats.date, DeviceStats.device_id) diff --git a/ogn_python/model/flights2d.py b/ogn_python/model/flights2d.py new file mode 100644 index 0000000..2dc3128 --- /dev/null +++ b/ogn_python/model/flights2d.py @@ -0,0 +1,27 @@ +from geoalchemy2.types import Geometry + +from ogn_python import db + + +class Flight2D(db.Model): + __tablename__ = "flights2d" + + date = db.Column(db.Date, primary_key=True) + flight_type = db.Column(db.SmallInteger, primary_key=True) + + path_wkt = db.Column('path', Geometry('MULTILINESTRING', srid=4326)) + path_simple_wkt = db.Column('path_simple', Geometry('MULTILINESTRING', srid=4326)) # this is the path simplified with ST_Simplify(path, 0.0001) + + # Relations + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True) + device = db.relationship('Device', foreign_keys=[device_id], backref='flights2d') + + def __repr__(self): + return "" % ( + self.date, + self.path_wkt, + self.path_simple_wkt) + + +db.Index('ix_flights2d_date_device_id', Flight2D.date, Flight2D.device_id) +#db.Index('ix_flights2d_date_path', Flight2D.date, Flight2D.path_wkt) --> CREATE INDEX ix_flights2d_date_path ON flights2d USING GIST("date", path) diff --git a/ogn/model/geo.py b/ogn_python/model/geo.py similarity index 100% rename from ogn/model/geo.py rename to ogn_python/model/geo.py diff --git a/ogn_python/model/logbook.py b/ogn_python/model/logbook.py new file mode 100644 index 0000000..40eef16 --- /dev/null +++ b/ogn_python/model/logbook.py @@ -0,0 +1,34 @@ +from sqlalchemy.ext.hybrid import hybrid_property + +from ogn_python import db + + +class Logbook(db.Model): + __tablename__ = 'logbook' + + id = db.Column(db.Integer, primary_key=True) + + reftime = db.Column(db.DateTime, index=True) + takeoff_timestamp = db.Column(db.DateTime) + takeoff_track = db.Column(db.SmallInteger) + landing_timestamp = db.Column(db.DateTime) + landing_track = db.Column(db.SmallInteger) + max_altitude = db.Column(db.Float(precision=2)) + + # Relations + takeoff_airport_id = db.Column(db.Integer, db.ForeignKey('airports.id', ondelete='CASCADE'), index=True) + takeoff_airport = db.relationship('Airport', foreign_keys=[takeoff_airport_id]) + + landing_airport_id = db.Column(db.Integer, db.ForeignKey('airports.id', ondelete='CASCADE'), index=True) + landing_airport = db.relationship('Airport', foreign_keys=[landing_airport_id]) + + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='CASCADE'), index=True) + device = db.relationship('Device', foreign_keys=[device_id], backref=db.backref('logbook', order_by='Logbook.reftime')) + + @hybrid_property + def duration(self): + return None if (self.landing_timestamp is None or self.takeoff_timestamp is None) else self.landing_timestamp - self.takeoff_timestamp + + @duration.expression + def duration(cls): + return case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != null() and cls.takeoff_timestamp != null()) diff --git a/ogn_python/model/receiver.py b/ogn_python/model/receiver.py new file mode 100644 index 0000000..97d5e71 --- /dev/null +++ b/ogn_python/model/receiver.py @@ -0,0 +1,33 @@ +from geoalchemy2.shape import to_shape +from geoalchemy2.types import Geometry + +from .geo import Location + +from ogn_python import db + + +class Receiver(db.Model): + __tablename__ = "receivers" + + id = db.Column(db.Integer, primary_key=True) + + location_wkt = db.Column('location', Geometry('POINT', srid=4326)) + altitude = db.Column(db.Float(precision=2)) + + name = db.Column(db.String(9), index=True) + firstseen = db.Column(db.DateTime, index=True) + lastseen = db.Column(db.DateTime, index=True) + version = db.Column(db.String) + platform = db.Column(db.String) + + # Relations + country_id = db.Column(db.Integer, db.ForeignKey('countries.gid', ondelete='SET NULL'), index=True) + country = db.relationship('Country', foreign_keys=[country_id], backref=db.backref('receivers', order_by='Receiver.name.asc()')) + + @property + def location(self): + if self.location_wkt is None: + return None + + coords = to_shape(self.location_wkt) + return Location(lat=coords.y, lon=coords.x) diff --git a/ogn/model/receiver_beacon.py b/ogn_python/model/receiver_beacon.py similarity index 70% rename from ogn/model/receiver_beacon.py rename to ogn_python/model/receiver_beacon.py index a342bb8..e50fdb5 100644 --- a/ogn/model/receiver_beacon.py +++ b/ogn_python/model/receiver_beacon.py @@ -1,8 +1,8 @@ -from sqlalchemy import Column, Float, String, Integer, ForeignKey, Index -from sqlalchemy.orm import relationship from sqlalchemy.sql import func from .beacon import Beacon +from ogn_python import db + class ReceiverBeacon(Beacon): __tablename__ = "receiver_beacons" @@ -12,34 +12,34 @@ class ReceiverBeacon(Beacon): ground_speed = None # ReceiverBeacon specific data - version = Column(String) - platform = Column(String) - cpu_load = Column(Float(precision=2)) - free_ram = Column(Float(precision=2)) - total_ram = Column(Float(precision=2)) - ntp_error = Column(Float(precision=2)) - rt_crystal_correction = Column(Float(precision=2)) - voltage = Column(Float(precision=2)) - amperage = Column(Float(precision=2)) - cpu_temp = Column(Float(precision=2)) - senders_visible = Column(Integer) - senders_total = Column(Integer) - rec_input_noise = Column(Float(precision=2)) - senders_signal = Column(Float(precision=2)) - senders_messages = Column(Integer) - good_senders_signal = Column(Float(precision=2)) - good_senders = Column(Integer) - good_and_bad_senders = Column(Integer) + version = db.Column(db.String) + platform = db.Column(db.String) + cpu_load = db.Column(db.Float(precision=2)) + free_ram = db.Column(db.Float(precision=2)) + total_ram = db.Column(db.Float(precision=2)) + ntp_error = db.Column(db.Float(precision=2)) + rt_crystal_correction = db.Column(db.Float(precision=2)) + voltage = db.Column(db.Float(precision=2)) + amperage = db.Column(db.Float(precision=2)) + cpu_temp = db.Column(db.Float(precision=2)) + senders_visible = db.Column(db.Integer) + senders_total = db.Column(db.Integer) + rec_input_noise = db.Column(db.Float(precision=2)) + senders_signal = db.Column(db.Float(precision=2)) + senders_messages = db.Column(db.Integer) + good_senders_signal = db.Column(db.Float(precision=2)) + good_senders = db.Column(db.Integer) + good_and_bad_senders = db.Column(db.Integer) # User comment: used for additional information like hardware configuration, web site, email address, ... user_comment = None # Relations - receiver_id = Column(Integer, ForeignKey('receivers.id', ondelete='SET NULL')) - receiver = relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_beacons') + receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL')) + receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref='receiver_beacons') # Multi-column indices - Index('ix_receiver_beacons_receiver_id_name', 'receiver_id', 'name') + db.Index('ix_receiver_beacons_receiver_id_name', 'receiver_id', 'name') def __repr__(self): return "" % ( @@ -125,4 +125,4 @@ class ReceiverBeacon(Beacon): int(self.good_and_bad_senders) if self.good_and_bad_senders else None] -Index('ix_receiver_beacons_date_receiver_id', func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id) +db.Index('ix_receiver_beacons_date_receiver_id', func.date(ReceiverBeacon.timestamp), ReceiverBeacon.receiver_id) diff --git a/ogn_python/model/receiver_coverage.py b/ogn_python/model/receiver_coverage.py new file mode 100644 index 0000000..5883039 --- /dev/null +++ b/ogn_python/model/receiver_coverage.py @@ -0,0 +1,23 @@ +from ogn_python import db + + +class ReceiverCoverage(db.Model): + __tablename__ = "receiver_coverages" + + location_mgrs_short = db.Column(db.String(9), primary_key=True) + date = db.Column(db.Date, primary_key=True) + + max_signal_quality = db.Column(db.Float) + max_altitude = db.Column(db.Float(precision=2)) + min_altitude = db.Column(db.Float(precision=2)) + aircraft_beacon_count = db.Column(db.Integer) + + device_count = db.Column(db.SmallInteger) + + # Relations + receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'), primary_key=True) + receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref=db.backref('receiver_coverages', order_by='ReceiverCoverage.date.asc()')) + + +db.Index('ix_receiver_coverages_date_receiver_id', ReceiverCoverage.date, ReceiverCoverage.receiver_id) +db.Index('ix_receiver_coverages_receiver_id_date', ReceiverCoverage.receiver_id, ReceiverCoverage.date) diff --git a/ogn_python/model/receiver_stats.py b/ogn_python/model/receiver_stats.py new file mode 100644 index 0000000..b66a795 --- /dev/null +++ b/ogn_python/model/receiver_stats.py @@ -0,0 +1,41 @@ +from geoalchemy2.types import Geometry + +from ogn_python import db + + +class ReceiverStats(db.Model): + __tablename__ = "receiver_stats" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + + # Static data + firstseen = db.Column(db.DateTime, index=True) + lastseen = db.Column(db.DateTime, index=True) + location_wkt = db.Column('location', Geometry('POINT', srid=4326)) + altitude = db.Column(db.Float(precision=2)) + version = db.Column(db.String) + platform = db.Column(db.String) + + # Statistic data + aircraft_beacon_count = db.Column(db.Integer) + aircraft_count = db.Column(db.SmallInteger) + max_distance = db.Column(db.Float) + quality = db.Column(db.Float(precision=2)) + + # Relation statistic data + quality_offset = db.Column(db.Float(precision=2)) + + # Ranking data + aircraft_beacon_count_ranking = db.Column(db.SmallInteger) + aircraft_count_ranking = db.Column(db.SmallInteger) + max_distance_ranking = db.Column(db.SmallInteger) + quality_ranking = db.Column(db.Integer) + + # Relations + receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'), index=True) + receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref=db.backref('stats', order_by='ReceiverStats.date.asc()')) + + +db.Index('ix_receiver_stats_date_receiver_id', ReceiverStats.date, ReceiverStats.receiver_id) diff --git a/ogn_python/model/relation_stats.py b/ogn_python/model/relation_stats.py new file mode 100644 index 0000000..413b755 --- /dev/null +++ b/ogn_python/model/relation_stats.py @@ -0,0 +1,29 @@ +from ogn_python import db + + +class RelationStats(db.Model): + __tablename__ = "relation_stats" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + + # Statistic data + quality = db.Column(db.Float(precision=2)) + beacon_count = db.Column(db.Integer) + + # Relations + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), index=True) + device = db.relationship('Device', foreign_keys=[device_id], backref='relation_stats') + receiver_id = db.Column(db.Integer, db.ForeignKey('receivers.id', ondelete='SET NULL'), index=True) + receiver = db.relationship('Receiver', foreign_keys=[receiver_id], backref='relation_stats') + + def __repr__(self): + return "" % ( + self.date, + self.quality, + self.beacon_count) + + +db.Index('ix_relation_stats_date_device_id', RelationStats.date, RelationStats.device_id, RelationStats.receiver_id) +db.Index('ix_relation_stats_date_receiver_id', RelationStats.date, RelationStats.receiver_id, RelationStats.device_id) diff --git a/ogn_python/model/takeoff_landing.py b/ogn_python/model/takeoff_landing.py new file mode 100644 index 0000000..a463c5c --- /dev/null +++ b/ogn_python/model/takeoff_landing.py @@ -0,0 +1,16 @@ +from ogn_python import db + + +class TakeoffLanding(db.Model): + __tablename__ = 'takeoff_landings' + + device_id = db.Column(db.Integer, db.ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True) + airport_id = db.Column(db.Integer, db.ForeignKey('airports.id', ondelete='SET NULL'), primary_key=True) + timestamp = db.Column(db.DateTime, primary_key=True) + + is_takeoff = db.Column(db.Boolean) + track = db.Column(db.SmallInteger) + + # Relations + airport = db.relationship('Airport', foreign_keys=[airport_id], backref='takeoff_landings') + device = db.relationship('Device', foreign_keys=[device_id], backref='takeoff_landings', order_by='TakeoffLanding.timestamp') diff --git a/ogn_python/navigation.py b/ogn_python/navigation.py new file mode 100644 index 0000000..c24434f --- /dev/null +++ b/ogn_python/navigation.py @@ -0,0 +1,14 @@ +from flask_nav import Nav +from flask_nav.elements import * + +nav = Nav() + +# registers the "top" menubar +nav.register_element('top_menubar', Navbar( + View('Home', 'index'), + View('Devices', 'devices'), + View('Receivers', 'receivers'), + View('Airports', 'airports'), + View('Logbook', 'logbook'), + View('Records', 'records'), +)) diff --git a/ogn_python/routes.py b/ogn_python/routes.py new file mode 100644 index 0000000..f13c3f9 --- /dev/null +++ b/ogn_python/routes.py @@ -0,0 +1,128 @@ +from flask import request, render_template + +from ogn_python import app +from ogn_python import db + +from ogn_python.model import * + + +@app.route('/') +@app.route('/index') +def index(): + return render_template('base.html') + + +@app.route('/devices', methods=['GET', 'POST']) +def devices(): + device_id = request.args.get('id') + if device_id: + device = db.session.query(Device) \ + .filter(Device.id == device_id) + + return render_template('device_detail.html', device=device) + else: + devices = db.session.query(Device) \ + .limit(100) + return render_template('devices.html', devices=devices) + + +@app.route('/receivers') +def receivers(): + receivers = db.session.query(Receiver) \ + .filter(Receiver.country != db.null()) \ + .order_by(Receiver.name) + return render_template('receivers.html', receivers=receivers) + + +@app.route('/airports') +def airports(): + page = request.args.get('page', 1, type=int) + + pagination = db.session.query(Airport) \ + .order_by(Airport.name) \ + .paginate(page, 20, False) + return render_template('airports.html', pagination=pagination) + + +@app.route('/logbook', methods=['GET', 'POST']) +def logbook(): + sel_country = request.args.get('country') + sel_airport = request.args.get('airport') + sel_date = request.args.get('date') + + sel_device_id = request.args.get('device_id') + + airport_ids_in_logbook = db.session.query(db.distinct(Logbook.takeoff_airport_id).label('id')) \ + .subquery() + + airports_in_logbook = db.session.query(Airport) \ + .filter(Airport.id == airport_ids_in_logbook.c.id) \ + .subquery() + + country_ids_in_logbook = db.session.query(db.distinct(Country.gid).label('id')) \ + .filter(Country.iso2 == airports_in_logbook.c.country_code) \ + .subquery() + + countries_avail = db.session.query(Country) \ + .filter(Country.gid == country_ids_in_logbook.c.id) \ + .order_by(Country.iso2) + + # Get airport selection list + if sel_country: + airports = db.session.query(Airport) \ + .filter(Airport.id == airport_ids_in_logbook.c.id) \ + .filter(Airport.country_code == sel_country) \ + .order_by(Airport.name) + else: + airports = [''] + + # Get date selection list + if sel_country and sel_airport: + dates = db.session.query(db.func.date(Logbook.reftime), db.func.count(Logbook.id)) \ + .filter(db.or_(Logbook.takeoff_airport_id == sel_airport, + Logbook.landing_airport_id == sel_airport)) \ + .group_by(db.func.date(Logbook.reftime)) \ + .order_by(db.func.date(Logbook.reftime)) + else: + dates = [''] + + # Get Logbook + filters = [] + if sel_date: + filters.append(db.func.date(Logbook.reftime) == sel_date) + + if sel_country and sel_airport: + filters.append(db.or_(Logbook.takeoff_airport_id == sel_airport, Logbook.landing_airport_id == sel_airport)) + + if sel_device_id: + filters.append(Logbook.device_id == sel_device_id) + + if len(filters) > 0: + logbook = db.session.query(Logbook.takeoff_timestamp, + db.func.round(Logbook.takeoff_track/10).label('takeoff_track'), + Logbook.landing_timestamp, + db.func.round(Logbook.landing_track/10).label('landing_track'), + Logbook.max_altitude, + DeviceInfo.aircraft, + DeviceInfo.registration, + DeviceInfo.competition) \ + .filter(*filters) \ + .filter(db.and_(Logbook.device_id == Device.id, Device.address == DeviceInfo.address)) \ + .order_by(Logbook.reftime) + else: + logbook = None + + return render_template('logbook.html', sel_country=sel_country, countries=countries_avail, sel_airport=sel_airport, airports=airports, sel_date=sel_date, dates=dates, logbook=logbook) + + +@app.route('/live') +def live(): + return render_template('ogn_live.jinja') + + +@app.route('/records') +def records(): + receiverstats = db.session.query(ReceiverStats) \ + .limit(10) + + return render_template('records.html', receiverstats=receiverstats) \ No newline at end of file diff --git a/ogn_python/static/css/flags/LICENSE b/ogn_python/static/css/flags/LICENSE new file mode 100644 index 0000000..9f7fd6c --- /dev/null +++ b/ogn_python/static/css/flags/LICENSE @@ -0,0 +1,11 @@ +License + +FamFamFam flag icons are "available for free use for any purpose with no requirement for attribution" + +Blogpotato.de flag icons are licensed under Creative Commons + + + +Author and license terms for Maxmind icon set are unknown. + +- See more at: https://www.flag-sprites.com/en_US/#sthash.cOJO8GvT.dpuf \ No newline at end of file diff --git a/ogn_python/static/css/flags/flags.css b/ogn_python/static/css/flags/flags.css new file mode 100644 index 0000000..104bb6d --- /dev/null +++ b/ogn_python/static/css/flags/flags.css @@ -0,0 +1,259 @@ +.flag { + width: 16px; + height: 11px; + background:url(flags.png) no-repeat +} + +.flag.flag-ad {background-position: -16px 0} +.flag.flag-ae {background-position: -32px 0} +.flag.flag-af {background-position: -48px 0} +.flag.flag-ag {background-position: -64px 0} +.flag.flag-ai {background-position: -80px 0} +.flag.flag-al {background-position: -96px 0} +.flag.flag-am {background-position: -112px 0} +.flag.flag-an {background-position: -128px 0} +.flag.flag-ao {background-position: -144px 0} +.flag.flag-ar {background-position: -160px 0} +.flag.flag-as {background-position: -176px 0} +.flag.flag-at {background-position: -192px 0} +.flag.flag-au {background-position: -208px 0} +.flag.flag-aw {background-position: -224px 0} +.flag.flag-az {background-position: -240px 0} +.flag.flag-ba {background-position: 0 -11px} +.flag.flag-bb {background-position: -16px -11px} +.flag.flag-bd {background-position: -32px -11px} +.flag.flag-be {background-position: -48px -11px} +.flag.flag-bf {background-position: -64px -11px} +.flag.flag-bg {background-position: -80px -11px} +.flag.flag-bh {background-position: -96px -11px} +.flag.flag-bi {background-position: -112px -11px} +.flag.flag-bj {background-position: -128px -11px} +.flag.flag-bm {background-position: -144px -11px} +.flag.flag-bn {background-position: -160px -11px} +.flag.flag-bo {background-position: -176px -11px} +.flag.flag-br {background-position: -192px -11px} +.flag.flag-bs {background-position: -208px -11px} +.flag.flag-bt {background-position: -224px -11px} +.flag.flag-bv {background-position: -240px -11px} +.flag.flag-bw {background-position: 0 -22px} +.flag.flag-by {background-position: -16px -22px} +.flag.flag-bz {background-position: -32px -22px} +.flag.flag-ca {background-position: -48px -22px} +.flag.flag-catalonia {background-position: -64px -22px} +.flag.flag-cd {background-position: -80px -22px} +.flag.flag-cf {background-position: -96px -22px} +.flag.flag-cg {background-position: -112px -22px} +.flag.flag-ch {background-position: -128px -22px} +.flag.flag-ci {background-position: -144px -22px} +.flag.flag-ck {background-position: -160px -22px} +.flag.flag-cl {background-position: -176px -22px} +.flag.flag-cm {background-position: -192px -22px} +.flag.flag-cn {background-position: -208px -22px} +.flag.flag-co {background-position: -224px -22px} +.flag.flag-cr {background-position: -240px -22px} +.flag.flag-cu {background-position: 0 -33px} +.flag.flag-cv {background-position: -16px -33px} +.flag.flag-cw {background-position: -32px -33px} +.flag.flag-cy {background-position: -48px -33px} +.flag.flag-cz {background-position: -64px -33px} +.flag.flag-de {background-position: -80px -33px} +.flag.flag-dj {background-position: -96px -33px} +.flag.flag-dk {background-position: -112px -33px} +.flag.flag-dm {background-position: -128px -33px} +.flag.flag-do {background-position: -144px -33px} +.flag.flag-dz {background-position: -160px -33px} +.flag.flag-ec {background-position: -176px -33px} +.flag.flag-ee {background-position: -192px -33px} +.flag.flag-eg {background-position: -208px -33px} +.flag.flag-eh {background-position: -224px -33px} +.flag.flag-england {background-position: -240px -33px} +.flag.flag-er {background-position: 0 -44px} +.flag.flag-es {background-position: -16px -44px} +.flag.flag-et {background-position: -32px -44px} +.flag.flag-eu {background-position: -48px -44px} +.flag.flag-fi {background-position: -64px -44px} +.flag.flag-fj {background-position: -80px -44px} +.flag.flag-fk {background-position: -96px -44px} +.flag.flag-fm {background-position: -112px -44px} +.flag.flag-fo {background-position: -128px -44px} +.flag.flag-fr {background-position: -144px -44px} +.flag.flag-ga {background-position: -160px -44px} +.flag.flag-gb {background-position: -176px -44px} +.flag.flag-gd {background-position: -192px -44px} +.flag.flag-ge {background-position: -208px -44px} +.flag.flag-gf {background-position: -224px -44px} +.flag.flag-gg {background-position: -240px -44px} +.flag.flag-gh {background-position: 0 -55px} +.flag.flag-gi {background-position: -16px -55px} +.flag.flag-gl {background-position: -32px -55px} +.flag.flag-gm {background-position: -48px -55px} +.flag.flag-gn {background-position: -64px -55px} +.flag.flag-gp {background-position: -80px -55px} +.flag.flag-gq {background-position: -96px -55px} +.flag.flag-gr {background-position: -112px -55px} +.flag.flag-gs {background-position: -128px -55px} +.flag.flag-gt {background-position: -144px -55px} +.flag.flag-gu {background-position: -160px -55px} +.flag.flag-gw {background-position: -176px -55px} +.flag.flag-gy {background-position: -192px -55px} +.flag.flag-hk {background-position: -208px -55px} +.flag.flag-hm {background-position: -224px -55px} +.flag.flag-hn {background-position: -240px -55px} +.flag.flag-hr {background-position: 0 -66px} +.flag.flag-ht {background-position: -16px -66px} +.flag.flag-hu {background-position: -32px -66px} +.flag.flag-ic {background-position: -48px -66px} +.flag.flag-id {background-position: -64px -66px} +.flag.flag-ie {background-position: -80px -66px} +.flag.flag-il {background-position: -96px -66px} +.flag.flag-im {background-position: -112px -66px} +.flag.flag-in {background-position: -128px -66px} +.flag.flag-io {background-position: -144px -66px} +.flag.flag-iq {background-position: -160px -66px} +.flag.flag-ir {background-position: -176px -66px} +.flag.flag-is {background-position: -192px -66px} +.flag.flag-it {background-position: -208px -66px} +.flag.flag-je {background-position: -224px -66px} +.flag.flag-jm {background-position: -240px -66px} +.flag.flag-jo {background-position: 0 -77px} +.flag.flag-jp {background-position: -16px -77px} +.flag.flag-ke {background-position: -32px -77px} +.flag.flag-kg {background-position: -48px -77px} +.flag.flag-kh {background-position: -64px -77px} +.flag.flag-ki {background-position: -80px -77px} +.flag.flag-km {background-position: -96px -77px} +.flag.flag-kn {background-position: -112px -77px} +.flag.flag-kp {background-position: -128px -77px} +.flag.flag-kr {background-position: -144px -77px} +.flag.flag-kurdistan {background-position: -160px -77px} +.flag.flag-kw {background-position: -176px -77px} +.flag.flag-ky {background-position: -192px -77px} +.flag.flag-kz {background-position: -208px -77px} +.flag.flag-la {background-position: -224px -77px} +.flag.flag-lb {background-position: -240px -77px} +.flag.flag-lc {background-position: 0 -88px} +.flag.flag-li {background-position: -16px -88px} +.flag.flag-lk {background-position: -32px -88px} +.flag.flag-lr {background-position: -48px -88px} +.flag.flag-ls {background-position: -64px -88px} +.flag.flag-lt {background-position: -80px -88px} +.flag.flag-lu {background-position: -96px -88px} +.flag.flag-lv {background-position: -112px -88px} +.flag.flag-ly {background-position: -128px -88px} +.flag.flag-ma {background-position: -144px -88px} +.flag.flag-mc {background-position: -160px -88px} +.flag.flag-md {background-position: -176px -88px} +.flag.flag-me {background-position: -192px -88px} +.flag.flag-mg {background-position: -208px -88px} +.flag.flag-mh {background-position: -224px -88px} +.flag.flag-mk {background-position: -240px -88px} +.flag.flag-ml {background-position: 0 -99px} +.flag.flag-mm {background-position: -16px -99px} +.flag.flag-mn {background-position: -32px -99px} +.flag.flag-mo {background-position: -48px -99px} +.flag.flag-mp {background-position: -64px -99px} +.flag.flag-mq {background-position: -80px -99px} +.flag.flag-mr {background-position: -96px -99px} +.flag.flag-ms {background-position: -112px -99px} +.flag.flag-mt {background-position: -128px -99px} +.flag.flag-mu {background-position: -144px -99px} +.flag.flag-mv {background-position: -160px -99px} +.flag.flag-mw {background-position: -176px -99px} +.flag.flag-mx {background-position: -192px -99px} +.flag.flag-my {background-position: -208px -99px} +.flag.flag-mz {background-position: -224px -99px} +.flag.flag-na {background-position: -240px -99px} +.flag.flag-nc {background-position: 0 -110px} +.flag.flag-ne {background-position: -16px -110px} +.flag.flag-nf {background-position: -32px -110px} +.flag.flag-ng {background-position: -48px -110px} +.flag.flag-ni {background-position: -64px -110px} +.flag.flag-nl {background-position: -80px -110px} +.flag.flag-no {background-position: -96px -110px} +.flag.flag-np {background-position: -112px -110px} +.flag.flag-nr {background-position: -128px -110px} +.flag.flag-nu {background-position: -144px -110px} +.flag.flag-nz {background-position: -160px -110px} +.flag.flag-om {background-position: -176px -110px} +.flag.flag-pa {background-position: -192px -110px} +.flag.flag-pe {background-position: -208px -110px} +.flag.flag-pf {background-position: -224px -110px} +.flag.flag-pg {background-position: -240px -110px} +.flag.flag-ph {background-position: 0 -121px} +.flag.flag-pk {background-position: -16px -121px} +.flag.flag-pl {background-position: -32px -121px} +.flag.flag-pm {background-position: -48px -121px} +.flag.flag-pn {background-position: -64px -121px} +.flag.flag-pr {background-position: -80px -121px} +.flag.flag-ps {background-position: -96px -121px} +.flag.flag-pt {background-position: -112px -121px} +.flag.flag-pw {background-position: -128px -121px} +.flag.flag-py {background-position: -144px -121px} +.flag.flag-qa {background-position: -160px -121px} +.flag.flag-re {background-position: -176px -121px} +.flag.flag-ro {background-position: -192px -121px} +.flag.flag-rs {background-position: -208px -121px} +.flag.flag-ru {background-position: -224px -121px} +.flag.flag-rw {background-position: -240px -121px} +.flag.flag-sa {background-position: 0 -132px} +.flag.flag-sb {background-position: -16px -132px} +.flag.flag-sc {background-position: -32px -132px} +.flag.flag-scotland {background-position: -48px -132px} +.flag.flag-sd {background-position: -64px -132px} +.flag.flag-se {background-position: -80px -132px} +.flag.flag-sg {background-position: -96px -132px} +.flag.flag-sh {background-position: -112px -132px} +.flag.flag-si {background-position: -128px -132px} +.flag.flag-sk {background-position: -144px -132px} +.flag.flag-sl {background-position: -160px -132px} +.flag.flag-sm {background-position: -176px -132px} +.flag.flag-sn {background-position: -192px -132px} +.flag.flag-so {background-position: -208px -132px} +.flag.flag-somaliland {background-position: -224px -132px} +.flag.flag-sr {background-position: -240px -132px} +.flag.flag-ss {background-position: 0 -143px} +.flag.flag-st {background-position: -16px -143px} +.flag.flag-sv {background-position: -32px -143px} +.flag.flag-sx {background-position: -48px -143px} +.flag.flag-sy {background-position: -64px -143px} +.flag.flag-sz {background-position: -80px -143px} +.flag.flag-tc {background-position: -96px -143px} +.flag.flag-td {background-position: -112px -143px} +.flag.flag-tf {background-position: -128px -143px} +.flag.flag-tg {background-position: -144px -143px} +.flag.flag-th {background-position: -160px -143px} +.flag.flag-tibet {background-position: -176px -143px} +.flag.flag-tj {background-position: -192px -143px} +.flag.flag-tk {background-position: -208px -143px} +.flag.flag-tl {background-position: -224px -143px} +.flag.flag-tm {background-position: -240px -143px} +.flag.flag-tn {background-position: 0 -154px} +.flag.flag-to {background-position: -16px -154px} +.flag.flag-tr {background-position: -32px -154px} +.flag.flag-tt {background-position: -48px -154px} +.flag.flag-tv {background-position: -64px -154px} +.flag.flag-tw {background-position: -80px -154px} +.flag.flag-tz {background-position: -96px -154px} +.flag.flag-ua {background-position: -112px -154px} +.flag.flag-ug {background-position: -128px -154px} +.flag.flag-um {background-position: -144px -154px} +.flag.flag-us {background-position: -160px -154px} +.flag.flag-uy {background-position: -176px -154px} +.flag.flag-uz {background-position: -192px -154px} +.flag.flag-va {background-position: -208px -154px} +.flag.flag-vc {background-position: -224px -154px} +.flag.flag-ve {background-position: -240px -154px} +.flag.flag-vg {background-position: 0 -165px} +.flag.flag-vi {background-position: -16px -165px} +.flag.flag-vn {background-position: -32px -165px} +.flag.flag-vu {background-position: -48px -165px} +.flag.flag-wales {background-position: -64px -165px} +.flag.flag-wf {background-position: -80px -165px} +.flag.flag-ws {background-position: -96px -165px} +.flag.flag-xk {background-position: -112px -165px} +.flag.flag-ye {background-position: -128px -165px} +.flag.flag-yt {background-position: -144px -165px} +.flag.flag-za {background-position: -160px -165px} +.flag.flag-zanzibar {background-position: -176px -165px} +.flag.flag-zm {background-position: -192px -165px} +.flag.flag-zw {background-position: -208px -165px} diff --git a/ogn_python/static/css/flags/flags.png b/ogn_python/static/css/flags/flags.png new file mode 100644 index 0000000..4a7e206 Binary files /dev/null and b/ogn_python/static/css/flags/flags.png differ diff --git a/ogn_python/static/img/Blank.gif b/ogn_python/static/img/Blank.gif new file mode 100644 index 0000000..2799b45 Binary files /dev/null and b/ogn_python/static/img/Blank.gif differ diff --git a/ogn_python/static/img/Transparent.gif b/ogn_python/static/img/Transparent.gif new file mode 100644 index 0000000..f191b28 Binary files /dev/null and b/ogn_python/static/img/Transparent.gif differ diff --git a/ogn_python/templates/airports.html b/ogn_python/templates/airports.html new file mode 100644 index 0000000..f23e6ee --- /dev/null +++ b/ogn_python/templates/airports.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block content %} + +
+
+

Airports

+
+ + + + + + + + {% for airport in pagination.items %} + + + + + {% endfor %} +
#NameCountry
{{ loop.index }} + {{ airport.name }}{{ airport.country_code }}
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/base.html b/ogn_python/templates/base.html new file mode 100644 index 0000000..94d6f82 --- /dev/null +++ b/ogn_python/templates/base.html @@ -0,0 +1,24 @@ +{% extends 'bootstrap/base.html' %} + +{% block title %} + {% if title %}{{ title }}{% else %}No page title{% endif %} +{% endblock %} + +{% block navbar %} +{{nav.top_menubar.render()}} +{% endblock %} + +{% block content %} +
+ {% with messages = get_flashed_messages() %} + {% if messages %} + {% for message in messages %} + + {% endfor %} + {% endif %} + {% endwith %} + + {# application content needs to be provided in the app_content block #} + {% block app_content %}{% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/device_detail.html b/ogn_python/templates/device_detail.html new file mode 100644 index 0000000..572fb86 --- /dev/null +++ b/ogn_python/templates/device_detail.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} + +{% block content %} + +
+
+

Devices

+
+ +{{ device.address }} + + + + + + + + + {% for entry in device.logbook %} + + + + + + + + + {% endfor %} +
Nr.TakeoffLandingAGL
{{ loop.index }}{% if entry.takeoff_timestamp is not none %} {{ entry.takeoff_timestamp.strftime('%H:%M') }} {% endif %}{% if entry.takeoff_track is not none %} {{ '%02d' | format(entry.takeoff_track) }} {% endif %}{% if entry.landing_timestamp is not none %} {{ entry.landing_timestamp.strftime('%H:%M') }} {% endif %}{% if entry.landing_track is not none %} {{ '%02d' | format(entry.landing_track) }} {% endif %}{% if entry.max_altitude is not none %} {{ entry.max_altitude }} {% endif %}
+ +
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/devices.html b/ogn_python/templates/devices.html new file mode 100644 index 0000000..e86c2d8 --- /dev/null +++ b/ogn_python/templates/devices.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} + +{% block content %} + +
+
+

Devices

+
+ + + + + + + + + + {% for device in devices %} + + + + + + {% endfor %} +
AddressAirportLast takeoff/landingLogbookSoftware version
{{ device.address }}{% if device.takeoff_landings %}{% set last_action = device.takeoff_landings|last %}{{ last_action.airport.name }}{% endif %} + {% if device.takeoff_landings %}{% set last_action = device.takeoff_landings|last %}{% if last_action.is_takeoff == True %}↗{% else %}↘{% endif %} @ {{ last_action.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}{% endif %} + {% if device.infos %}{% set info = device.infos|first %}{{ info.registration }} {% else %} - {% endif %}{% if device.software_version is not none %}{% if device.software_version < 6.6 %}

{{ device.software_version }}

{% else %}{{ device.software_version }}{% endif %}{% else %} - {% endif %}
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/logbook.html b/ogn_python/templates/logbook.html new file mode 100644 index 0000000..b011b2b --- /dev/null +++ b/ogn_python/templates/logbook.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block content %} + +
+
+

Logbook

+
+ +
+
+ + + +
+
+ + +{% if logbook is not none %} + + + + + + + + + + {% for entry in logbook %} + + + + + + + + + + + {% endfor %} +
Nr.AircraftTypeTakeoffLandingAGL
{{ loop.index }}{{ entry.registration }}{{ entry.aircraft }}{% if entry.takeoff_timestamp is not none %} {{ entry.takeoff_timestamp.strftime('%H:%M') }} {% endif %}{% if entry.takeoff_track is not none %} {{ '%02d' | format(entry.takeoff_track) }} {% endif %}{% if entry.landing_timestamp is not none %} {{ entry.landing_timestamp.strftime('%H:%M') }} {% endif %}{% if entry.landing_track is not none %} {{ '%02d' | format(entry.landing_track) }} {% endif %}{% if entry.max_altitude is not none %} {{ entry.max_altitude }} {% endif %}
+{% endif %} +
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/ogn_live.html b/ogn_python/templates/ogn_live.html new file mode 100644 index 0000000..cc08c84 --- /dev/null +++ b/ogn_python/templates/ogn_live.html @@ -0,0 +1,81 @@ + + + + + + + Spot the gliders! + + + + + + + + + + + + +
+
+
+
+
+
+
+ + + \ No newline at end of file diff --git a/ogn_python/templates/receivers.html b/ogn_python/templates/receivers.html new file mode 100644 index 0000000..8789902 --- /dev/null +++ b/ogn_python/templates/receivers.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block content %} + + + +
+
+

Receivers

+
+ + + + + + + + {% for receiver in receivers %} + + + + + + {% endfor %} +
NameCountryAltitude
{{ receiver.name }}{{ receiver.country.iso2 }}{{ receiver.altitude }}
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn_python/templates/records.html b/ogn_python/templates/records.html new file mode 100644 index 0000000..1e5b5dc --- /dev/null +++ b/ogn_python/templates/records.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} + +{% block content %} + + + +
+
+

Devices

+
+ + + + + + + + + {% for receiverstat in receiverstats %} + + + + + + + {% endfor %} +
ReceiverCountryAircraftsBeacons
{{ receiverstat.receiver.name }}{{ receiverstat.receiver.country.iso2 }}{{ receiverstat.aircraft_count }}{{ receiverstat.beacon_count }}
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/ogn/utils.py b/ogn_python/utils.py similarity index 100% rename from ogn/utils.py rename to ogn_python/utils.py diff --git a/setup.py b/setup.py index 6f716b8..1b88a59 100644 --- a/setup.py +++ b/setup.py @@ -32,11 +32,13 @@ setup( keywords='gliding ogn', packages=find_packages(exclude=['tests', 'tests.*']), install_requires=[ - 'SQLAlchemy==1.2.12', + 'Flask==1.0.2', + 'flask-sqlalchemy==2.3.2', + 'Flask-Migrate==2.3.1', + 'flask-bootstrap==3.3.7.1', + 'flask-nav==0.6', 'geopy==1.17.0', - 'manage.py==0.2.10', 'celery[redis]==4.2.1', - 'alembic==1.0.0', 'aerofiles==0.4.1', 'geoalchemy2==0.5.0', 'shapely>=1.5.17,<1.6', @@ -44,7 +46,7 @@ setup( 'psycopg2-binary==2.7.6.1', 'mgrs==1.3.5', 'xmlunittest==0.5.0', - 'tqdm==4.28.1' + 'tqdm==4.28.1', ], extras_require={ 'dev': [ diff --git a/tests/backend/test_backends.py b/tests/backend/test_backends.py index 06312cd..d9ea103 100644 --- a/tests/backend/test_backends.py +++ b/tests/backend/test_backends.py @@ -5,11 +5,11 @@ from datetime import datetime from xmlunittest import XmlTestMixin -from ogn.model import AircraftBeacon, Receiver, Device, DeviceInfo +from ogn_python.model import AircraftBeacon, Receiver, Device, DeviceInfo -from ogn.backend.liveglidernet import rec, lxml -from ogn.backend.ognrange import stations2_filtered_pl -from ogn.model.aircraft_type import AircraftType +from ogn_python.backend.liveglidernet import rec, lxml +from ogn_python.backend.ognrange import stations2_filtered_pl +from ogn_python.model.aircraft_type import AircraftType class TestDB(unittest.TestCase, XmlTestMixin): @@ -19,11 +19,11 @@ class TestDB(unittest.TestCase, XmlTestMixin): def setUp(self): os.environ['OGN_CONFIG_MODULE'] = 'config.test' - from ogn.commands.dbutils import engine, session + from ogn_python.commands.dbutils import engine, session self.session = session self.engine = engine - from ogn.commands.database import init + from ogn_python.commands.database import init init() # Prepare Beacons diff --git a/tests/base.py b/tests/base.py index 220945a..b36cba2 100644 --- a/tests/base.py +++ b/tests/base.py @@ -10,14 +10,14 @@ class TestBaseDB(unittest.TestCase): @classmethod def setUpClass(cls): - from ogn.commands.dbutils import engine, session + from ogn_python.commands.dbutils import engine, session cls.session = session cls.engine = engine - from ogn.commands.database import drop + from ogn_python.commands.database import drop drop(sure='y') - from ogn.commands.database import init + from ogn_python.commands.database import init init() def setUp(self): diff --git a/tests/collect/test_database.py b/tests/collect/test_database.py index 4fba4ed..8df47b0 100644 --- a/tests/collect/test_database.py +++ b/tests/collect/test_database.py @@ -2,8 +2,8 @@ import unittest from tests.base import TestBaseDB -from ogn.model import AircraftBeacon, ReceiverBeacon, Device, Receiver -from ogn.collect.database import add_missing_devices, add_missing_receivers, upsert +from ogn_python.model import AircraftBeacon, ReceiverBeacon, Device, Receiver +from ogn_python.collect.database import add_missing_devices, add_missing_receivers, upsert class TestDatabase(TestBaseDB): diff --git a/tests/collect/test_logbook.py b/tests/collect/test_logbook.py index 7f021e2..3ca7eef 100644 --- a/tests/collect/test_logbook.py +++ b/tests/collect/test_logbook.py @@ -2,8 +2,8 @@ import unittest from tests.base import TestBaseDB -from ogn.model import Logbook, Airport, Device, TakeoffLanding -from ogn.collect.logbook import update_logbook +from ogn_python.model import Logbook, Airport, Device, TakeoffLanding +from ogn_python.collect.logbook import update_logbook class TestLogbook(TestBaseDB): diff --git a/tests/collect/test_ognrange.py b/tests/collect/test_ognrange.py index 99f87a2..297bc29 100644 --- a/tests/collect/test_ognrange.py +++ b/tests/collect/test_ognrange.py @@ -3,8 +3,8 @@ from datetime import date from tests.base import TestBaseDB -from ogn.model import AircraftBeacon, Receiver, ReceiverCoverage, Device -from ogn.collect.ognrange import create_receiver_coverage +from ogn_python.model import AircraftBeacon, Receiver, ReceiverCoverage, Device +from ogn_python.collect.ognrange import create_receiver_coverage class TestOGNrange(TestBaseDB): diff --git a/tests/collect/test_stats.py b/tests/collect/test_stats.py index 91b6f20..cd11b41 100644 --- a/tests/collect/test_stats.py +++ b/tests/collect/test_stats.py @@ -3,9 +3,9 @@ from datetime import datetime, date from tests.base import TestBaseDB -from ogn.model import AircraftBeacon, ReceiverBeacon, Receiver, Device, DeviceStats +from ogn_python.model import AircraftBeacon, ReceiverBeacon, Receiver, Device, DeviceStats -from ogn.collect.stats import create_device_stats +from ogn_python.collect.stats import create_device_stats class TestStats(TestBaseDB): diff --git a/tests/collect/test_takeoff_landing.py b/tests/collect/test_takeoff_landing.py index 8141a8d..67624c3 100644 --- a/tests/collect/test_takeoff_landing.py +++ b/tests/collect/test_takeoff_landing.py @@ -2,9 +2,9 @@ import unittest from tests.base import TestBaseDB -from ogn.model import TakeoffLanding +from ogn_python.model import TakeoffLanding -from ogn.collect.takeoff_landings import update_takeoff_landings +from ogn_python.collect.takeoff_landings import update_takeoff_landings class TestTakeoffLanding(TestBaseDB): diff --git a/tests/commands/test_database.py b/tests/commands/test_database.py index ef9ec09..6e705fa 100644 --- a/tests/commands/test_database.py +++ b/tests/commands/test_database.py @@ -3,8 +3,8 @@ import os from tests.base import TestBaseDB -from ogn.model import DeviceInfo -from ogn.commands.database import import_file +from ogn_python.model import DeviceInfo +from ogn_python.commands.database import import_file class TestDatabase(TestBaseDB): diff --git a/tests/gateway/test_saver.py b/tests/gateway/test_saver.py index 11ae227..90929a7 100644 --- a/tests/gateway/test_saver.py +++ b/tests/gateway/test_saver.py @@ -2,7 +2,7 @@ import datetime import unittest from unittest.mock import MagicMock -from ogn.gateway.process_tools import DbSaver +from ogn_python.gateway.process_tools import DbSaver class DbSaverTest(unittest.TestCase): diff --git a/tests/test_utils.py b/tests/test_utils.py index 7c89d01..166cf26 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,8 +2,8 @@ import os import unittest from datetime import date -from ogn.model import AircraftType -from ogn.utils import get_days, get_ddb, get_trackable, get_airports +from ogn_python.model import AircraftType +from ogn_python.utils import get_days, get_ddb, get_trackable, get_airports class TestStringMethods(unittest.TestCase):