From b263b00f7c668055195ef465f52516ecf04d86af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Sun, 22 Nov 2020 08:55:19 +0100 Subject: [PATCH] PEP8 violations --- app/__init__.py | 8 ++- app/collect/flights.py | 5 +- app/collect/gateway.py | 4 +- app/collect/logbook.py | 82 ++++++++++++------------- app/collect/timescaledb_views.py | 18 +++--- app/commands/database.py | 4 +- app/commands/export.py | 28 ++++----- app/commands/gateway.py | 35 +---------- app/gateway/beacon_conversion.py | 1 - app/gateway/message_handling.py | 36 +++++------ app/gateway/process_tools.py | 1 + app/main/__init__.py | 2 +- app/main/jinja_filters.py | 5 +- app/main/matplotlib_service.py | 16 ++--- app/main/routes.py | 34 +++++----- app/model/logbook.py | 4 +- app/model/receiver.py | 11 ++-- app/model/sender_direction_statistic.py | 1 + app/model/sender_info.py | 1 + app/model/sender_position.py | 3 +- app/model/sender_position_statistic.py | 1 + app/tasks/orm_tasks.py | 5 +- app/tasks/sql_tasks.py | 12 ++-- app/utils.py | 5 +- config.py | 16 +++-- setup.cfg | 2 +- tests/collect/test_logbook.py | 1 - 27 files changed, 166 insertions(+), 175 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 59e43c2..c594b83 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -26,23 +26,25 @@ def create_app(config_name='default'): configuration = configs[config_name] app.config.from_object(configuration) app.config.from_envvar("OGN_CONFIG_MODULE", silent=True) - + # Initialize other things bootstrap.init_app(app) db.init_app(app) migrate.init_app(app, db) cache.init_app(app) redis_client.init_app(app) - + init_celery(app) register_blueprints(app) - + return app + def register_blueprints(app): from app.main import bp as bp_main app.register_blueprint(bp_main) + def init_celery(app=None): app = app or create_app(os.getenv('FLASK_CONFIG') or 'default') celery.conf.broker_url = app.config['BROKER_URL'] diff --git a/app/collect/flights.py b/app/collect/flights.py index 922ee4a..4e79e58 100644 --- a/app/collect/flights.py +++ b/app/collect/flights.py @@ -6,6 +6,7 @@ NOTHING = "" CONTEST_RELEVANT = "AND agl < 1000" LOW_PASS = "AND agl < 50 and ground_speed > 250" + def compute_flights(date, flight_type=0): if flight_type == 0: filter = NOTHING @@ -66,6 +67,7 @@ def compute_flights(date, flight_type=0): db.session.execute(query) db.session.commit() + def compute_gaps(date): date_str = date.strftime("%Y-%m-%d") @@ -105,9 +107,10 @@ def compute_gaps(date): db.session.execute(query) db.session.commit() + if __name__ == '__main__': from app import create_app app = create_app() with app.app_context(): result = compute_flights(date=date(2020, 10, 28)) - print(result) \ No newline at end of file + print(result) diff --git a/app/collect/gateway.py b/app/collect/gateway.py index f1850d1..3cb47e7 100644 --- a/app/collect/gateway.py +++ b/app/collect/gateway.py @@ -4,8 +4,10 @@ from flask import current_app from app import redis_client from app.gateway.message_handling import sender_position_csv_strings_to_db, receiver_position_csv_strings_to_db, receiver_status_csv_strings_to_db + def transfer_from_redis_to_database(): - unmapping = lambda s: s[0].decode('utf-8') + def unmapping(string): + return string[0].decode('utf-8') receiver_status_data = list(map(unmapping, redis_client.zpopmin('receiver_status', 100000))) receiver_position_data = list(map(unmapping, redis_client.zpopmin('receiver_position', 100000))) diff --git a/app/collect/logbook.py b/app/collect/logbook.py index b213fe4..128dde4 100644 --- a/app/collect/logbook.py +++ b/app/collect/logbook.py @@ -46,7 +46,7 @@ def update_takeoff_landings(start, end): .filter(db.between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION))) .subquery() ) - + # make a query with current, previous and next position sq2 = db.session.query( sq.c.name, @@ -75,11 +75,11 @@ def update_takeoff_landings(start, end): # consider only positions between start and end and with predecessor and successor and limit distance and duration between points sq3 = ( db.session.query(sq2) - .filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null())) - .filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS)) - .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION)) - .filter(db.between(sq2.c.timestamp, start, end)) - .subquery() + .filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null())) + .filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS)) + .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION)) + .filter(db.between(sq2.c.timestamp, start, end)) + .subquery() ) # find possible takeoffs and landings @@ -131,8 +131,8 @@ def update_takeoff_landings(start, end): # ... add the country takeoff_landing_query = ( db.session.query(sq6.c.timestamp, sq6.c.track, sq6.c.is_takeoff, sq6.c.sender_id, sq6.c.airport_id, Country.gid) - .join(Country, sq6.c.country_code==Country.iso2, isouter=True) - .subquery() + .join(Country, sq6.c.country_code == Country.iso2, isouter=True) + .subquery() ) # ... and save them @@ -156,7 +156,7 @@ def update_logbook(offset_days=None): # limit time range to given date and set window partition and window order if offset_days: - (start, end) = date_to_timestamps(datetime.utcnow()-timedelta(days=offset_days)) + (start, end) = date_to_timestamps(datetime.utcnow() - timedelta(days=offset_days)) else: (start, end) = date_to_timestamps(datetime.utcnow().date()) pa = TakeoffLanding.sender_id @@ -181,7 +181,6 @@ def update_logbook(offset_days=None): db.func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"), db.func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next") ) - #.filter(between(TakeoffLanding.timestamp, start, end)) .subquery() ) @@ -195,7 +194,7 @@ def update_logbook(offset_days=None): ) .filter(sq.c.is_takeoff == db.true()) .filter(db.or_(sq.c.is_takeoff_next == db.true(), sq.c.is_takeoff_next == db.null())) - .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.takeoff_timestamp==sq.c.timestamp, Logbook.takeoff_airport_id==sq.c.airport_id)).exists()) + .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == sq.c.sender_id, Logbook.takeoff_timestamp == sq.c.timestamp, Logbook.takeoff_airport_id == sq.c.airport_id)).exists()) ) ins = insert(Logbook).from_select( ( @@ -220,7 +219,7 @@ def update_logbook(offset_days=None): ) .filter(db.or_(sq.c.is_takeoff_prev == db.false(), sq.c.is_takeoff_prev == db.null())) .filter(sq.c.is_takeoff == db.false()) - .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.landing_timestamp==sq.c.timestamp, Logbook.landing_airport_id==sq.c.airport_id)).exists()) + .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == sq.c.sender_id, Logbook.landing_timestamp == sq.c.timestamp, Logbook.landing_airport_id == sq.c.airport_id)).exists()) ) ins = insert(Logbook).from_select( ( @@ -253,9 +252,9 @@ def update_logbook(offset_days=None): # insert (new) flights new_flights_query = ( - db.session.query(complete_flight_query) \ - .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==complete_flight_query.c.sender_id, Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id)).exists()) - .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==complete_flight_query.c.sender_id, Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id)).exists()) + db.session.query(complete_flight_query) + .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.landing_timestamp == complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id == complete_flight_query.c.landing_airport_id)).exists()) + .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.takeoff_timestamp == complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id == complete_flight_query.c.takeoff_airport_id)).exists()) ) ins = insert(Logbook).from_select( ( @@ -276,17 +275,16 @@ def update_logbook(offset_days=None): # update existing landing with takeoff from complete flight upd = db.update(Logbook) \ .where(db.and_( - Logbook.sender_id==complete_flight_query.c.sender_id, - Logbook.takeoff_timestamp==db.null(), - Logbook.takeoff_airport_id==db.null(), - Logbook.landing_timestamp!=db.null(), - Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp, - Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id + Logbook.sender_id == complete_flight_query.c.sender_id, + Logbook.takeoff_timestamp == db.null(), + Logbook.takeoff_airport_id == db.null(), + Logbook.landing_timestamp != db.null(), + Logbook.landing_timestamp == complete_flight_query.c.landing_timestamp, + Logbook.landing_airport_id == complete_flight_query.c.landing_airport_id )) \ .values(takeoff_timestamp=complete_flight_query.c.takeoff_timestamp, takeoff_track=complete_flight_query.c.takeoff_track, - takeoff_airport_id=complete_flight_query.c.takeoff_airport_id - ) + takeoff_airport_id=complete_flight_query.c.takeoff_airport_id) result = db.session.execute(upd) current_app.logger.debug(f"Updated {result.rowcount} takeoffs to complete flights") db.session.commit() @@ -294,17 +292,16 @@ def update_logbook(offset_days=None): # update existing takeoff with landing from complete flight upd = db.update(Logbook) \ .where(db.and_( - Logbook.sender_id==complete_flight_query.c.sender_id, - Logbook.takeoff_timestamp!=db.null(), - Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp, - Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id, - Logbook.landing_timestamp==db.null(), - Logbook.landing_airport_id==db.null() + Logbook.sender_id == complete_flight_query.c.sender_id, + Logbook.takeoff_timestamp != db.null(), + Logbook.takeoff_timestamp == complete_flight_query.c.takeoff_timestamp, + Logbook.takeoff_airport_id == complete_flight_query.c.takeoff_airport_id, + Logbook.landing_timestamp == db.null(), + Logbook.landing_airport_id == db.null() )) \ .values(landing_timestamp=complete_flight_query.c.landing_timestamp, landing_track=complete_flight_query.c.landing_track, - landing_airport_id=complete_flight_query.c.landing_airport_id - ) + landing_airport_id=complete_flight_query.c.landing_airport_id) result = db.session.execute(upd) current_app.logger.debug(f"Updated {result.rowcount} landings to complete flights") db.session.commit() @@ -312,11 +309,10 @@ def update_logbook(offset_days=None): return - def update_max_altitudes(): MAX_UPDATES = 60 - query = f""" + query = """ UPDATE logbooks SET max_altitude = sq2.max_altitude FROM ( @@ -347,6 +343,7 @@ def update_max_altitudes(): return update_counter + def update_max_altitudes_orm(): """Add max altitudes in logbook when flight is complete (takeoff and landing).""" @@ -354,17 +351,17 @@ def update_max_altitudes_orm(): logbook_entries = ( db.session.query(Logbook.id, Sender.name) - .filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null())) - .filter(Logbook.sender_id == Sender.id) - .limit(1) - .subquery() + .filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null())) + .filter(Logbook.sender_id == Sender.id) + .limit(1) + .subquery() ) max_altitudes = ( db.session.query(logbook_entries.c.id, db.func.max(SenderPosition.altitude).label("max_altitude")) - .filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name)) - .group_by(Logbook.id) - .subquery() + .filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name)) + .group_by(Logbook.id) + .subquery() ) update_logbooks = db.session.query(Logbook).filter(Logbook.id == max_altitudes.c.id).update({Logbook.max_altitude: max_altitudes.c.max_altitude}, synchronize_session="fetch") @@ -374,11 +371,12 @@ def update_max_altitudes_orm(): finish_message = "Logbook (altitude): {} entries updated.".format(update_logbooks) return finish_message + if __name__ == '__main__': from app import create_app app = create_app() with app.app_context(): - #result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0)) - #result = update_logbook() + result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0)) + result = update_logbook() result = update_max_altitudes_orm() print(result) diff --git a/app/collect/timescaledb_views.py b/app/collect/timescaledb_views.py index b8d5411..b577b04 100644 --- a/app/collect/timescaledb_views.py +++ b/app/collect/timescaledb_views.py @@ -3,8 +3,9 @@ from app.utils import get_sql_trustworthy SQL_TRUSTWORTHY = get_sql_trustworthy(source_table_alias='sp') + def create_views(): - db.session.execute(f""" + db.session.execute(""" DROP VIEW IF EXISTS receiver_ranking CASCADE; CREATE VIEW receiver_ranking AS @@ -23,7 +24,7 @@ def create_views(): ORDER BY max_distance DESC; """) - db.session.execute(f""" + db.session.execute(""" DROP VIEW IF EXISTS sender_ranking CASCADE; CREATE VIEW sender_ranking AS @@ -44,6 +45,7 @@ def create_views(): db.session.commit() + def create_timescaledb_views(): # 1. Since the reference_timestamps are strictly increasing we can set # the parameter 'refresh_lag' to a very short time so the materialization @@ -51,11 +53,11 @@ def create_timescaledb_views(): # 2. The feature realtime aggregation from TimescaleDB is quite time consuming. # So we set materialized_only=true - ### Sender statistics + # --- Sender statistics --- # These stats will be used in the daily ranking, so we make the bucket < 1d db.session.execute(f""" DROP VIEW IF EXISTS sender_stats_1h CASCADE; - + CREATE VIEW sender_stats_1h WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='5 minutes') AS SELECT @@ -90,7 +92,7 @@ def create_timescaledb_views(): GROUP BY bucket, sp.name, is_trustworthy; """) - ### Receiver statistics + # --- Receiver statistics --- # These stats will be used in the daily ranking, so we make the bucket < 1d db.session.execute(f""" DROP VIEW IF EXISTS receiver_stats_1h CASCADE; @@ -128,8 +130,8 @@ def create_timescaledb_views(): FROM sender_positions AS sp GROUP BY bucket, sp.receiver_name, is_trustworthy; """) - - ### Relation statistics (sender <-> receiver) + + # --- Relation statistics (sender <-> receiver) --- # these stats will be used on a >= 1d basis, so we make the bucket = 1d db.session.execute(f""" DROP VIEW IF EXISTS relation_stats_1d CASCADE; @@ -162,4 +164,4 @@ class MyView(db.Model): autoload=True, autoload_with=db.engine ) -""" \ No newline at end of file +""" diff --git a/app/commands/database.py b/app/commands/database.py index e404f99..2901364 100644 --- a/app/commands/database.py +++ b/app/commands/database.py @@ -119,6 +119,7 @@ def import_airports(path="tests/SeeYou.cup"): db.session.commit() print("Imported {} airports.".format(len(airports))) + @user_cli.command("create_timescaledb_views") def cmd_create_timescaledb_views(): """Create TimescaleDB views.""" @@ -126,11 +127,10 @@ def cmd_create_timescaledb_views(): create_timescaledb_views() print("Done") + @user_cli.command("create_views") def cmd_create_views(): """Create views.""" create_views() print("Done") - - diff --git a/app/commands/export.py b/app/commands/export.py index d129cf9..a6f07c4 100644 --- a/app/commands/export.py +++ b/app/commands/export.py @@ -15,6 +15,7 @@ from app import db user_cli = AppGroup("export") user_cli.help = "Export data in several file formats." + @user_cli.command("debug_sql") @click.argument("start") @click.argument("end") @@ -24,7 +25,7 @@ def debug_sql(start, end, name): # First: get all the positions (and the receiver names for later) sql_sender_positions = f""" - SELECT reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl + SELECT reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl FROM sender_positions WHERE reference_timestamp BETWEEN '{start}' AND '{end}' AND name = '{name}' ORDER BY reference_timestamp; @@ -38,7 +39,7 @@ def debug_sql(start, end, name): receiver_names.append("'" + row[2] + "'") row = [f"'{r}'" if r else "DEFAULT" for r in row] sender_position_values.append(f"({','.join(row)})") - + # Second: get the receivers sql_receivers = f""" SELECT name, location @@ -50,8 +51,8 @@ def debug_sql(start, end, name): results = db.session.execute(sql_receivers) for row in results: row = [f"'{r}'" if r else "DEFAULT" for r in row] - receiver_values.append(f"({','.join(row)})") - + receiver_values.append(f"({','.join(row)})") + # Third: get the airports sql_airports = f""" SELECT DISTINCT a.name, a.location, a.altitude, a.style, a.border @@ -66,23 +67,22 @@ def debug_sql(start, end, name): results = db.session.execute(sql_airports) for row in results: row = [f"'{r}'" if r else "DEFAULT" for r in row] - airport_values.append(f"({','.join(row)})") + airport_values.append(f"({','.join(row)})") # Last: write all into file with open(f'{start}_{end}_{name}.sql', 'w') as file: - file.write(f'/*\n') - file.write(f'OGN Python SQL Export\n') + file.write('/*\n') + file.write('OGN Python SQL Export\n') file.write(f'Created by: {os.getlogin()}\n') file.write(f'Created at: {datetime.datetime.utcnow()}\n') - file.write(f'*/\n\n') - + file.write('*/\n\n') file.write("INSERT INTO airports(name, location, altitude, style, border) VALUES\n") file.write(',\n'.join(airport_values) + ';\n\n') file.write("INSERT INTO receivers(name, location) VALUES\n") file.write(',\n'.join(receiver_values) + ';\n\n') - + file.write("INSERT INTO sender_positions(reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl) VALUES\n") file.write(',\n'.join(sender_position_values) + ';\n\n') @@ -139,7 +139,7 @@ def igc(address, date): return try: - sender = db.session.query(Sender).filter(Sender.address==address).one() + sender = db.session.query(Sender).filter(Sender.address == address).one() except NoResultFound as e: print(f"No data for '{address}' in the DB") return @@ -173,9 +173,9 @@ def igc(address, date): points = ( db.session.query(SenderPosition) - .filter(db.between(SenderPosition.reference_timestamp, f"{date} 00:00:00", f"{date} 23:59:59")) - .filter(SenderPosition.name == sender.name) - .order_by(SenderPosition.timestamp) + .filter(db.between(SenderPosition.reference_timestamp, f"{date} 00:00:00", f"{date} 23:59:59")) + .filter(SenderPosition.name == sender.name) + .order_by(SenderPosition.timestamp) ) for point in points: diff --git a/app/commands/gateway.py b/app/commands/gateway.py index 8d8319a..d22cf87 100644 --- a/app/commands/gateway.py +++ b/app/commands/gateway.py @@ -84,7 +84,7 @@ def transfer(): """Transfer data from redis to the database.""" transfer_from_redis_to_database() - + @user_cli.command("printout") @click.option("--aprs_filter", default='') @@ -101,36 +101,3 @@ def printout(aprs_filter): current_app.logger.warning("\nStop ogn gateway") client.disconnect() - -@user_cli.command("convert") -@click.argument("path") -def file_import(path): - """Convert APRS logfiles into csv files for fast bulk import.""" - - for (root, dirs, files) in os.walk(path): - for file in sorted(files): - print(file) - convert(os.path.join(root, file)) - - -@user_cli.command("calculate") -@click.argument("path") -def file_calculate(path): - """Import csv files, calculate geographic features (distance, radial, agl, ...) and make data distinct.""" - - file_tuples = [] - for (root, dirs, files) in os.walk(path): - for file in sorted(files): - if file.startswith('aircraft_beacons') and file.endswith('.csv.gz'): - ab_filename = os.path.join(root, file) - rb_filename = os.path.join(root, 'receiver' + file[8:]) - target_filename = os.path.join(root, file + '2') - if os.path.isfile(target_filename): - print("Outputfile {} already exists. Skipping".format(target_filename)) - else: - file_tuples.append((ab_filename, rb_filename, target_filename)) - - pbar = tqdm(file_tuples) - for file_tuple in pbar: - pbar.set_description("Converting {}".format(file_tuple[0])) - calculate(file_tuple[0], file_tuple[1], file_tuple[2]) diff --git a/app/gateway/beacon_conversion.py b/app/gateway/beacon_conversion.py index 58e6408..7925bf2 100644 --- a/app/gateway/beacon_conversion.py +++ b/app/gateway/beacon_conversion.py @@ -40,7 +40,6 @@ def aprs_string_to_message(aprs_string): bearing = int(message['bearing']) message['bearing'] = bearing if bearing < 360 else 0 - if "aircraft_type" in message: message["aircraft_type"] = AircraftType(message["aircraft_type"]) if message["aircraft_type"] in AircraftType.list() else AircraftType.UNKNOWN diff --git a/app/gateway/message_handling.py b/app/gateway/message_handling.py index 6cee18c..bef9986 100644 --- a/app/gateway/message_handling.py +++ b/app/gateway/message_handling.py @@ -18,11 +18,11 @@ SENDER_POSITION_BEACON_FIELDS = [ "receiver_name", "timestamp", "location", - + "track", "ground_speed", "altitude", - + "address_type", "aircraft_type", "stealth", @@ -38,7 +38,7 @@ SENDER_POSITION_BEACON_FIELDS = [ "hardware_version", "real_address", "signal_power", - + "distance", "bearing", "normalized_quality", @@ -66,7 +66,7 @@ RECEIVER_POSITION_BEACON_FIELDS = [ RECEIVER_STATUS_BEACON_FIELDS = [ "reference_timestamp", - + "name", "dstcall", "receiver_name", @@ -90,7 +90,7 @@ def sender_position_message_to_csv_string(message, none_character=''): csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23},{24},{25},{26},{27},{28},{29},{30}\n".format( message['reference_timestamp'], - + message['name'], message['dstcall'], message['relay'] if 'relay' in message and message['relay'] else none_character, @@ -101,8 +101,8 @@ def sender_position_message_to_csv_string(message, none_character=''): message['track'] if 'track' in message and message['track'] else none_character, message['ground_speed'] if 'ground_speed' in message and message['ground_speed'] else none_character, int(message['altitude']) if message['altitude'] else none_character, - - message['address_type'] if 'address_type' in message and message['address_type'] else none_character, #10 + + message['address_type'] if 'address_type' in message and message['address_type'] else none_character, # 10 message['aircraft_type'].name if 'aircraft_type' in message and message['aircraft_type'] else AircraftType.UNKNOWN.name, message['stealth'] if 'stealth' in message and message['stealth'] else none_character, message['address'] if 'address' in message and message['address'] else none_character, @@ -112,12 +112,12 @@ def sender_position_message_to_csv_string(message, none_character=''): message['error_count'] if 'error_count' in message and message['error_count'] else none_character, message['frequency_offset'] if 'frequency_offset' in message and message['frequency_offset'] else none_character, message['gps_quality_horizontal'] if 'gps_quality_horizontal' in message and message['gps_quality_horizontal'] else none_character, - message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character, #20 + message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character, # 20 message['software_version'] if 'software_version' in message and message['software_version'] else none_character, message['hardware_version'] if 'hardware_version' in message and message['hardware_version'] else none_character, message['real_address'] if 'real_address' in message and message['real_address'] else none_character, message['signal_power'] if 'signal_power' in message and message['signal_power'] else none_character, - + message['distance'] if 'distance' in message and message['distance'] else none_character, message['bearing'] if 'bearing' in message and message['bearing'] else none_character, message['normalized_quality'] if 'normalized_quality' in message and message['normalized_quality'] else none_character, @@ -132,7 +132,7 @@ def sender_position_message_to_csv_string(message, none_character=''): def receiver_position_message_to_csv_string(message, none_character=''): csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n".format( message['reference_timestamp'], - + message['name'], message['dstcall'], message['receiver_name'], @@ -180,7 +180,7 @@ def sender_position_csv_strings_to_db(lines): cursor.execute(f"CREATE TEMPORARY TABLE {tmp_tablename} (LIKE sender_positions) ON COMMIT DROP;") cursor.copy_from(file=string_buffer, table=tmp_tablename, sep=",", columns=SENDER_POSITION_BEACON_FIELDS) - + # Update agl cursor.execute(f""" UPDATE {tmp_tablename} AS tmp @@ -238,7 +238,7 @@ def sender_position_csv_strings_to_db(lines): """) # Update sender_infos FK -> senders - cursor.execute(f""" + cursor.execute(""" UPDATE sender_infos AS si SET sender_id = s.id FROM senders AS s @@ -316,7 +316,7 @@ def receiver_position_csv_strings_to_db(lines): tmp.name, tmp.timestamp, tmp.location, - + tmp.altitude, tmp.agl @@ -340,18 +340,18 @@ def receiver_position_csv_strings_to_db(lines): """) # Update receiver country - cursor.execute(f""" + cursor.execute(""" UPDATE receivers AS r - SET + SET country_id = c.gid FROM countries AS c WHERE r.country_id IS NULL AND ST_Within(r.location, c.geom); """) # Update receiver airport - cursor.execute(f""" + cursor.execute(""" UPDATE receivers AS r - SET + SET airport_id = ( SELECT id FROM airports AS a @@ -400,7 +400,7 @@ def receiver_status_csv_strings_to_db(lines): tmp.name, tmp.timestamp, - + tmp.version, tmp.platform, diff --git a/app/gateway/process_tools.py b/app/gateway/process_tools.py index c8e565b..1ec2bc1 100644 --- a/app/gateway/process_tools.py +++ b/app/gateway/process_tools.py @@ -37,6 +37,7 @@ class Timer(object): print("[{}]".format(self.name)) print("Elapsed: {}".format(time.time() - self.tstart)) + def export_to_path(path): connection = db.engine.raw_connection() cursor = connection.cursor() diff --git a/app/main/__init__.py b/app/main/__init__.py index fe53f94..5829369 100644 --- a/app/main/__init__.py +++ b/app/main/__init__.py @@ -3,4 +3,4 @@ from flask import Blueprint bp = Blueprint("main", __name__) import app.main.routes -import app.main.jinja_filters \ No newline at end of file +import app.main.jinja_filters diff --git a/app/main/jinja_filters.py b/app/main/jinja_filters.py index 615ac81..6cdb960 100644 --- a/app/main/jinja_filters.py +++ b/app/main/jinja_filters.py @@ -6,6 +6,7 @@ import time import datetime import math + @bp.app_template_filter() def timestamp_to_status(timestamp): if datetime.datetime.utcnow() - timestamp < datetime.timedelta(minutes=10): @@ -15,6 +16,7 @@ def timestamp_to_status(timestamp): else: return 'OFFLINE' + @bp.app_template_filter() def to_html_link(obj): if isinstance(obj, Airport): @@ -40,6 +42,7 @@ def to_html_link(obj): else: raise NotImplementedError("cant apply filter 'to_html_link' to object {type(obj)}") + @bp.app_template_filter() def to_ordinal(rad): deg = math.degrees(rad) @@ -58,4 +61,4 @@ def to_ordinal(rad): elif deg >= 247.5 and deg < 292.5: return "E" elif deg >= 292.5 and deg < 337.5: - return "NE" \ No newline at end of file + return "NE" diff --git a/app/main/matplotlib_service.py b/app/main/matplotlib_service.py index fad6064..7b4cf54 100644 --- a/app/main/matplotlib_service.py +++ b/app/main/matplotlib_service.py @@ -1,10 +1,11 @@ from app import db -from app.model import * +from app.model import SenderDirectionStatistic import random import numpy as np import matplotlib.pyplot as plt from matplotlib.figure import Figure + def create_range_figure2(sender_id): fig = Figure() axis = fig.add_subplot(1, 1, 1) @@ -14,6 +15,7 @@ def create_range_figure2(sender_id): return fig + def create_range_figure(sender_id): sds = db.session.query(SenderDirectionStatistic) \ .filter(SenderDirectionStatistic.sender_id == sender_id) \ @@ -24,11 +26,11 @@ def create_range_figure(sender_id): fig = Figure() direction_data = sds.direction_data - max_range = max([r['max_range']/1000.0 for r in direction_data]) + max_range = max([r['max_range'] / 1000.0 for r in direction_data]) - theta = np.array([i['direction']/180*np.pi for i in direction_data]) - radii = np.array([i['max_range']/1000 if i['max_range'] > 0 else 0 for i in direction_data]) - width = np.array([13/180*np.pi for i in direction_data]) + theta = np.array([i['direction'] / 180 * np.pi for i in direction_data]) + radii = np.array([i['max_range'] / 1000 if i['max_range'] > 0 else 0 for i in direction_data]) + width = np.array([13 / 180 * np.pi for i in direction_data]) colors = plt.cm.viridis(radii / max_range) ax = fig.add_subplot(111, projection='polar') @@ -38,5 +40,5 @@ def create_range_figure(sender_id): ax.set_theta_direction(-1) fig.suptitle(f"Range between sender '{sds.sender.name}' and receiver '{sds.receiver.name}'") - - return fig \ No newline at end of file + + return fig diff --git a/app/main/routes.py b/app/main/routes.py index 529cd01..67ea3d0 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -25,7 +25,7 @@ def get_used_countries(): @cache.memoize() def get_used_airports_by_country(sel_country): - query = db.session.query(Airport).filter(Airport.country_code == sel_country).filter(TakeoffLanding.airport_id==Airport.id).filter(TakeoffLanding.country_id == Country.gid).order_by(Airport.name).distinct(Airport.name) + query = db.session.query(Airport).filter(Airport.country_code == sel_country).filter(TakeoffLanding.airport_id == Airport.id).filter(TakeoffLanding.country_id == Country.gid).order_by(Airport.name).distinct(Airport.name) return [used_airport for used_airport in query] @@ -45,17 +45,18 @@ def get_dates_for_airport(sel_airport): @bp.route("/") @bp.route("/index.html") def index(): - today_beginning = datetime.combine(date.today(), time()) - - senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen>=today_beginning).one()[0] - receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen>=today_beginning).one()[0] - takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==True)).one()[0] - landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==False)).one()[0] - sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date==date.today()).one()[0] + today_beginning = datetime.combine(date.today(), time()) + + senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen >= today_beginning).one()[0] + receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen >= today_beginning).one()[0] + takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is True)).one()[0] + landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is False)).one()[0] + sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date == date.today()).one()[0] sender_positions_total = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).one()[0] last_logbook_entries = db.session.query(Logbook).order_by(Logbook.reference_timestamp.desc()).limit(10) - return render_template("index.html", + return render_template( + "index.html", senders_today=senders_today, receivers_today=receivers_today, takeoffs_today=takeoffs_today, @@ -80,13 +81,14 @@ def sender_detail(): return render_template("sender_detail.html", title="Sender", sender=sender) + @bp.route("/range_view.png") def range_view(): import io from flask import Response from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas - + sender_id = request.args.get("sender_id") fig = create_range_figure(sender_id) @@ -210,24 +212,28 @@ def download_flight(): return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain") + @bp.route("/sender_ranking.html") def sender_ranking(): sender_statistics = db.session.query(SenderStatistic) \ - .filter(db.and_(SenderStatistic.date==date.today(), SenderStatistic.is_trustworthy==True)) \ + .filter(db.and_(SenderStatistic.date == date.today(), SenderStatistic.is_trustworthy is True)) \ .order_by(SenderStatistic.max_distance.desc()) \ .all() - return render_template("sender_ranking.html", + return render_template( + "sender_ranking.html", title="Sender Ranking", ranking=sender_statistics) + @bp.route("/receiver_ranking.html") def receiver_ranking(): receiver_statistics = db.session.query(ReceiverStatistic) \ - .filter(db.and_(ReceiverStatistic.date==date.today(), ReceiverStatistic.is_trustworthy==True)) \ + .filter(db.and_(ReceiverStatistic.date == date.today(), ReceiverStatistic.is_trustworthy is True)) \ .order_by(ReceiverStatistic.max_distance.desc()) \ .all() - return render_template("receiver_ranking.html", + return render_template( + "receiver_ranking.html", title="Receiver Ranking", ranking=receiver_statistics) diff --git a/app/model/logbook.py b/app/model/logbook.py index 748a4ae..c294a9e 100644 --- a/app/model/logbook.py +++ b/app/model/logbook.py @@ -37,7 +37,7 @@ class Logbook(db.Model): @duration.expression def duration(cls): return db.case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != db.null() and cls.takeoff_timestamp != db.null()) - + @hybrid_property def reference_timestamp(self): return self.takeoff_timestamp if self.takeoff_timestamp is not None else self.landing_timestamp @@ -50,7 +50,7 @@ class Logbook(db.Model): # FIXME: does not work... # FIXME: this does not throw an error as the __table_args__ above, but there is no index created -#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})" +#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})" #Index("idx_logbook_reference_timestamp", _wrapped_case) # TODO: diff --git a/app/model/receiver.py b/app/model/receiver.py index 67f24ee..45a0a38 100644 --- a/app/model/receiver.py +++ b/app/model/receiver.py @@ -47,11 +47,10 @@ class Receiver(db.Model): def airports_nearby(self): query = ( db.session.query(Airport, db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt), db.func.st_azimuth(self.location_wkt, Airport.location_wkt)) - .filter(db.func.st_contains(db.func.st_buffer(Airport.location_wkt, 1), self.location_wkt)) - .filter(Airport.style.in_((2,4,5))) - .order_by(db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt).asc()) - .limit(5) + .filter(db.func.st_contains(db.func.st_buffer(Airport.location_wkt, 1), self.location_wkt)) + .filter(Airport.style.in_((2, 4, 5))) + .order_by(db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt).asc()) + .limit(5) ) - airports = [(airport,distance,azimuth) for airport, distance, azimuth in query] + airports = [(airport, distance, azimuth) for airport, distance, azimuth in query] return airports - diff --git a/app/model/sender_direction_statistic.py b/app/model/sender_direction_statistic.py index 96b759a..d468989 100644 --- a/app/model/sender_direction_statistic.py +++ b/app/model/sender_direction_statistic.py @@ -2,6 +2,7 @@ from app import db from sqlalchemy.dialects.postgresql import JSON + class SenderDirectionStatistic(db.Model): __tablename__ = "sender_direction_statistics" diff --git a/app/model/sender_info.py b/app/model/sender_info.py index 8c067b1..38e8f2b 100644 --- a/app/model/sender_info.py +++ b/app/model/sender_info.py @@ -4,6 +4,7 @@ from .aircraft_type import AircraftType #from sqlalchemy.dialects.postgresql import ENUM + class SenderInfo(db.Model): __tablename__ = "sender_infos" diff --git a/app/model/sender_position.py b/app/model/sender_position.py index caf9a24..914736a 100644 --- a/app/model/sender_position.py +++ b/app/model/sender_position.py @@ -48,7 +48,7 @@ class SenderPosition(db.Model): hardware_version = db.Column(db.SmallInteger) real_address = db.Column(db.String(6)) signal_power = db.Column(db.Float(precision=2)) - + #proximity = None # Calculated values (from parser) @@ -60,4 +60,3 @@ class SenderPosition(db.Model): location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool agl = db.Column(db.Float(precision=2)) - diff --git a/app/model/sender_position_statistic.py b/app/model/sender_position_statistic.py index a433227..8a92780 100644 --- a/app/model/sender_position_statistic.py +++ b/app/model/sender_position_statistic.py @@ -4,6 +4,7 @@ from .aircraft_type import AircraftType from sqlalchemy.dialects.postgresql import ENUM + class SenderPositionStatistic(db.Model): __tablename__ = "sender_position_statistics" diff --git a/app/tasks/orm_tasks.py b/app/tasks/orm_tasks.py index 125d3f9..b1b5a57 100644 --- a/app/tasks/orm_tasks.py +++ b/app/tasks/orm_tasks.py @@ -9,13 +9,14 @@ from app.collect.gateway import transfer_from_redis_to_database from app import db, celery + @celery.task(name="transfer_to_database") def transfer_to_database(): """Transfer APRS data from Redis to database.""" - + result = transfer_from_redis_to_database() return result - + @celery.task(name="update_takeoff_landings") def update_takeoff_landings(last_minutes): diff --git a/app/tasks/sql_tasks.py b/app/tasks/sql_tasks.py index 1cb2d2d..294acba 100644 --- a/app/tasks/sql_tasks.py +++ b/app/tasks/sql_tasks.py @@ -10,7 +10,7 @@ def update_statistics(date_str=None): if date_str is None: date_str = datetime.utcnow().strftime("%Y-%m-%d") - # Update relation statistics + # Update relation statistics db.session.execute(f""" DELETE FROM relation_statistics WHERE date = '{date_str}'; @@ -22,7 +22,7 @@ def update_statistics(date_str=None): tmp.receiver_id, is_trustworthy, - + MAX(tmp.max_distance) AS max_distance, MAX(tmp.max_normalized_quality) AS max_normalized_quality, SUM(tmp.messages_count) AS messages_count, @@ -43,7 +43,7 @@ def update_statistics(date_str=None): tmp.sender_id, is_trustworthy, - + MAX(tmp.max_distance) AS max_distance, MAX(tmp.max_normalized_quality) AS max_normalized_quality, SUM(tmp.messages_count) AS messages_count, @@ -65,7 +65,7 @@ def update_statistics(date_str=None): tmp.receiver_id, is_trustworthy, - + MAX(tmp.max_distance) AS max_distance, MAX(tmp.max_normalized_quality) AS max_normalized_quality, SUM(tmp.messages_count) AS messages_count, @@ -84,7 +84,7 @@ def update_sender_direction_statistics(): """ Update sender_direction_statistics.""" db.session.execute(""" - DELETE FROM sender_direction_statistics; + DELETE FROM sender_direction_statistics; INSERT INTO sender_direction_statistics(sender_id, receiver_id, directions_count, messages_count, direction_data) SELECT @@ -93,7 +93,7 @@ def update_sender_direction_statistics(): COUNT(sq2.*) AS directions_count, SUM(sq2.messages_count) AS messages_count, json_agg(json_build_object('direction', direction, 'messages_count', messages_count, 'max_range', max_range)) AS direction_data - FROM ( + FROM ( SELECT sq.sender_id, sq.receiver_id, diff --git a/app/utils.py b/app/utils.py index b6c59fe..51f0054 100644 --- a/app/utils.py +++ b/app/utils.py @@ -137,8 +137,9 @@ def open_file(filename): f = open(filename, "rt", encoding="latin-1") return f + def get_sql_trustworthy(source_table_alias): - MIN_DISTANCE = 1000 + MIN_DISTANCE = 1000 MAX_DISTANCE = 640000 MAX_NORMALIZED_QUALITY = 40 # this is enough for > 640km MAX_ERROR_COUNT = 5 @@ -149,4 +150,4 @@ def get_sql_trustworthy(source_table_alias): AND ({source_table_alias}.normalized_quality IS NOT NULL AND {source_table_alias}.normalized_quality < {MAX_NORMALIZED_QUALITY}) AND ({source_table_alias}.error_count IS NULL OR {source_table_alias}.error_count < {MAX_ERROR_COUNT}) AND ({source_table_alias}.climb_rate IS NULL OR {source_table_alias}.climb_rate BETWEEN -{MAX_CLIMB_RATE} AND {MAX_CLIMB_RATE}) - """ \ No newline at end of file + """ diff --git a/config.py b/config.py index fbd25d9..9589113 100644 --- a/config.py +++ b/config.py @@ -1,25 +1,27 @@ import os + class BaseConfig: SECRET_KEY = "i-like-ogn" - + # Flask-Cache stuff CACHE_TYPE = "simple" CACHE_DEFAULT_TIMEOUT = 300 - + # Redis stuff REDIS_URL = "redis://localhost:6379/0" - + # Celery stuff BROKER_URL = os.environ.get("BROKER_URL", REDIS_URL) CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", REDIS_URL) APRS_USER = "OGNPYTHON" + class DefaultConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "postgresql://postgres:postgres@localhost:5432/ogn") SQLALCHEMY_TRACK_MODIFICATIONS = False - + # Celery beat stuff from celery.schedules import crontab from datetime import timedelta @@ -33,17 +35,19 @@ class DefaultConfig(BaseConfig): "update_ddb_daily": {"task": "import_ddb", "schedule": timedelta(days=1)}, #"update_logbook_max_altitude": {"task": "update_logbook_max_altitude", "schedule": timedelta(minutes=1), "kwargs": {"offset_days": 0}}, - + #"purge_old_data": {"task": "purge_old_data", "schedule": timedelta(hours=1), "kwargs": {"max_hours": 48}}, } + class DevelopmentConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5432/ogn_test" SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_ECHO = False + configs = { 'default': DefaultConfig, 'development': DevelopmentConfig, 'testing': DevelopmentConfig -} \ No newline at end of file +} diff --git a/setup.cfg b/setup.cfg index 8d174ac..c37dcc1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [flake8] -ignore = F401, F841, E402, E501, E126 +ignore = F401, F841, E402, E501, E126, E265 diff --git a/tests/collect/test_logbook.py b/tests/collect/test_logbook.py index ffc7243..a6fefe3 100644 --- a/tests/collect/test_logbook.py +++ b/tests/collect/test_logbook.py @@ -76,7 +76,6 @@ class TestLogbook(TestBaseDB): self.assertEqual(entries[0].takeoff_airport_id, self.koenigsdorf.id) self.assertEqual(entries[0].landing_airport_id, self.koenigsdorf.id) - @unittest.skip('needs information about airport timezone') def test_takeoff_and_landing_on_different_days(self): db.session.add(self.takeoff_koenigsdorf_dd0815)