PEP8 violations

app/__init__.py

@@ -26,23 +26,25 @@ def create_app(config_name='default'):
     configuration = configs[config_name]
     app.config.from_object(configuration)
     app.config.from_envvar("OGN_CONFIG_MODULE", silent=True)
-    
+
     # Initialize other things
     bootstrap.init_app(app)
     db.init_app(app)
     migrate.init_app(app, db)
     cache.init_app(app)
     redis_client.init_app(app)
-    
+
     init_celery(app)
     register_blueprints(app)
-    
+
     return app
 
+
 def register_blueprints(app):
     from app.main import bp as bp_main
     app.register_blueprint(bp_main)
 
+
 def init_celery(app=None):
     app = app or create_app(os.getenv('FLASK_CONFIG') or 'default')
     celery.conf.broker_url = app.config['BROKER_URL']

app/collect/flights.py

@@ -6,6 +6,7 @@ NOTHING = ""
 CONTEST_RELEVANT = "AND agl < 1000"
 LOW_PASS = "AND agl < 50 and ground_speed > 250"
 
+
 def compute_flights(date, flight_type=0):
     if flight_type == 0:
         filter = NOTHING
@@ -66,6 +67,7 @@ def compute_flights(date, flight_type=0):
     db.session.execute(query)
     db.session.commit()
 
+
 def compute_gaps(date):
     date_str = date.strftime("%Y-%m-%d")
 
@@ -105,9 +107,10 @@ def compute_gaps(date):
     db.session.execute(query)
     db.session.commit()
 
+
 if __name__ == '__main__':
     from app import create_app
     app = create_app()
     with app.app_context():
         result = compute_flights(date=date(2020, 10, 28))
-        print(result)
+        print(result)

app/collect/gateway.py

@@ -4,8 +4,10 @@ from flask import current_app
 from app import redis_client
 from app.gateway.message_handling import sender_position_csv_strings_to_db, receiver_position_csv_strings_to_db, receiver_status_csv_strings_to_db
 
+
 def transfer_from_redis_to_database():
-    unmapping = lambda s: s[0].decode('utf-8')
+    def unmapping(string):
+        return string[0].decode('utf-8')
 
     receiver_status_data = list(map(unmapping, redis_client.zpopmin('receiver_status', 100000)))
     receiver_position_data = list(map(unmapping, redis_client.zpopmin('receiver_position', 100000)))

app/collect/logbook.py

@@ -46,7 +46,7 @@ def update_takeoff_landings(start, end):
         .filter(db.between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION)))
         .subquery()
     )
-    
+
     # make a query with current, previous and next position
     sq2 = db.session.query(
         sq.c.name,
@@ -75,11 +75,11 @@ def update_takeoff_landings(start, end):
     # consider only positions between start and end and with predecessor and successor and limit distance and duration between points
     sq3 = (
         db.session.query(sq2)
-            .filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null()))
+        .filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null()))
-            .filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS))
+        .filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS))
-            .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION))
+        .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION))
-            .filter(db.between(sq2.c.timestamp, start, end))
+        .filter(db.between(sq2.c.timestamp, start, end))
-            .subquery()
+        .subquery()
     )
 
     # find possible takeoffs and landings
@@ -131,8 +131,8 @@ def update_takeoff_landings(start, end):
     # ... add the country
     takeoff_landing_query = (
         db.session.query(sq6.c.timestamp, sq6.c.track, sq6.c.is_takeoff, sq6.c.sender_id, sq6.c.airport_id, Country.gid)
-            .join(Country,  sq6.c.country_code==Country.iso2, isouter=True)
+        .join(Country, sq6.c.country_code == Country.iso2, isouter=True)
-            .subquery()
+        .subquery()
     )
 
     # ... and save them
@@ -156,7 +156,7 @@ def update_logbook(offset_days=None):
 
     # limit time range to given date and set window partition and window order
     if offset_days:
-        (start, end) = date_to_timestamps(datetime.utcnow()-timedelta(days=offset_days))
+        (start, end) = date_to_timestamps(datetime.utcnow() - timedelta(days=offset_days))
     else:
         (start, end) = date_to_timestamps(datetime.utcnow().date())
     pa = TakeoffLanding.sender_id
@@ -181,7 +181,6 @@ def update_logbook(offset_days=None):
             db.func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"),
             db.func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next")
         )
-        #.filter(between(TakeoffLanding.timestamp, start, end))
         .subquery()
     )
 
@@ -195,7 +194,7 @@ def update_logbook(offset_days=None):
         )
         .filter(sq.c.is_takeoff == db.true())
         .filter(db.or_(sq.c.is_takeoff_next == db.true(), sq.c.is_takeoff_next == db.null()))
-        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.takeoff_timestamp==sq.c.timestamp, Logbook.takeoff_airport_id==sq.c.airport_id)).exists())
+        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == sq.c.sender_id, Logbook.takeoff_timestamp == sq.c.timestamp, Logbook.takeoff_airport_id == sq.c.airport_id)).exists())
     )
     ins = insert(Logbook).from_select(
         (
@@ -220,7 +219,7 @@ def update_logbook(offset_days=None):
         )
         .filter(db.or_(sq.c.is_takeoff_prev == db.false(), sq.c.is_takeoff_prev == db.null()))
         .filter(sq.c.is_takeoff == db.false())
-        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.landing_timestamp==sq.c.timestamp, Logbook.landing_airport_id==sq.c.airport_id)).exists())
+        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == sq.c.sender_id, Logbook.landing_timestamp == sq.c.timestamp, Logbook.landing_airport_id == sq.c.airport_id)).exists())
     )
     ins = insert(Logbook).from_select(
         (
@@ -253,9 +252,9 @@ def update_logbook(offset_days=None):
 
     # insert (new) flights
     new_flights_query = (
-        db.session.query(complete_flight_query) \
+        db.session.query(complete_flight_query)
-            .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==complete_flight_query.c.sender_id, Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id)).exists())
+        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.landing_timestamp == complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id == complete_flight_query.c.landing_airport_id)).exists())
-            .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==complete_flight_query.c.sender_id, Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id)).exists())
+        .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.takeoff_timestamp == complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id == complete_flight_query.c.takeoff_airport_id)).exists())
     )
     ins = insert(Logbook).from_select(
         (
@@ -276,17 +275,16 @@ def update_logbook(offset_days=None):
     # update existing landing with takeoff from complete flight
     upd = db.update(Logbook) \
         .where(db.and_(
-            Logbook.sender_id==complete_flight_query.c.sender_id,
+            Logbook.sender_id == complete_flight_query.c.sender_id,
-            Logbook.takeoff_timestamp==db.null(),
+            Logbook.takeoff_timestamp == db.null(),
-            Logbook.takeoff_airport_id==db.null(),
+            Logbook.takeoff_airport_id == db.null(),
-            Logbook.landing_timestamp!=db.null(),
+            Logbook.landing_timestamp != db.null(),
-            Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp,
+            Logbook.landing_timestamp == complete_flight_query.c.landing_timestamp,
-            Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id
+            Logbook.landing_airport_id == complete_flight_query.c.landing_airport_id
         )) \
         .values(takeoff_timestamp=complete_flight_query.c.takeoff_timestamp,
                 takeoff_track=complete_flight_query.c.takeoff_track,
-                takeoff_airport_id=complete_flight_query.c.takeoff_airport_id
+                takeoff_airport_id=complete_flight_query.c.takeoff_airport_id)
-        )
     result = db.session.execute(upd)
     current_app.logger.debug(f"Updated {result.rowcount} takeoffs to complete flights")
     db.session.commit()
@@ -294,17 +292,16 @@ def update_logbook(offset_days=None):
     # update existing takeoff with landing from complete flight
     upd = db.update(Logbook) \
         .where(db.and_(
-            Logbook.sender_id==complete_flight_query.c.sender_id,
+            Logbook.sender_id == complete_flight_query.c.sender_id,
-            Logbook.takeoff_timestamp!=db.null(),
+            Logbook.takeoff_timestamp != db.null(),
-            Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp,
+            Logbook.takeoff_timestamp == complete_flight_query.c.takeoff_timestamp,
-            Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id,
+            Logbook.takeoff_airport_id == complete_flight_query.c.takeoff_airport_id,
-            Logbook.landing_timestamp==db.null(),
+            Logbook.landing_timestamp == db.null(),
-            Logbook.landing_airport_id==db.null()
+            Logbook.landing_airport_id == db.null()
         )) \
         .values(landing_timestamp=complete_flight_query.c.landing_timestamp,
                 landing_track=complete_flight_query.c.landing_track,
-                landing_airport_id=complete_flight_query.c.landing_airport_id
+                landing_airport_id=complete_flight_query.c.landing_airport_id)
-        )
     result = db.session.execute(upd)
     current_app.logger.debug(f"Updated {result.rowcount} landings to complete flights")
     db.session.commit()
@@ -312,11 +309,10 @@ def update_logbook(offset_days=None):
     return
 
 
-
 def update_max_altitudes():
     MAX_UPDATES = 60
 
-    query = f"""
+    query = """
         UPDATE logbooks
             SET max_altitude = sq2.max_altitude
         FROM (
@@ -347,6 +343,7 @@ def update_max_altitudes():
 
     return update_counter
 
+
 def update_max_altitudes_orm():
     """Add max altitudes in logbook when flight is complete (takeoff and landing)."""
 
@@ -354,17 +351,17 @@ def update_max_altitudes_orm():
 
     logbook_entries = (
         db.session.query(Logbook.id, Sender.name)
-            .filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null()))
+        .filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null()))
-            .filter(Logbook.sender_id == Sender.id)
+        .filter(Logbook.sender_id == Sender.id)
-            .limit(1)
+        .limit(1)
-            .subquery()
+        .subquery()
     )
 
     max_altitudes = (
         db.session.query(logbook_entries.c.id, db.func.max(SenderPosition.altitude).label("max_altitude"))
-            .filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name))
+        .filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name))
-            .group_by(Logbook.id)
+        .group_by(Logbook.id)
-            .subquery()
+        .subquery()
     )
 
     update_logbooks = db.session.query(Logbook).filter(Logbook.id == max_altitudes.c.id).update({Logbook.max_altitude: max_altitudes.c.max_altitude}, synchronize_session="fetch")
@@ -374,11 +371,12 @@ def update_max_altitudes_orm():
     finish_message = "Logbook (altitude): {} entries updated.".format(update_logbooks)
     return finish_message
 
+
 if __name__ == '__main__':
     from app import create_app
     app = create_app()
     with app.app_context():
-        #result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0))
+        result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0))
-        #result = update_logbook()
+        result = update_logbook()
         result = update_max_altitudes_orm()
         print(result)

app/collect/timescaledb_views.py

@@ -3,8 +3,9 @@ from app.utils import get_sql_trustworthy
 
 SQL_TRUSTWORTHY = get_sql_trustworthy(source_table_alias='sp')
 
+
 def create_views():
-    db.session.execute(f"""
+    db.session.execute("""
         DROP VIEW IF EXISTS receiver_ranking CASCADE;
 
         CREATE VIEW receiver_ranking AS
@@ -23,7 +24,7 @@ def create_views():
         ORDER BY max_distance DESC;
     """)
 
-    db.session.execute(f"""
+    db.session.execute("""
         DROP VIEW IF EXISTS sender_ranking CASCADE;
 
         CREATE VIEW sender_ranking AS
@@ -44,6 +45,7 @@ def create_views():
 
     db.session.commit()
 
+
 def create_timescaledb_views():
     # 1. Since the reference_timestamps are strictly increasing we can set
     #    the parameter 'refresh_lag' to a very short time so the materialization
@@ -51,11 +53,11 @@ def create_timescaledb_views():
     # 2. The feature realtime aggregation from TimescaleDB is quite time consuming.
     #    So we set materialized_only=true
 
-    ### Sender statistics
+    # --- Sender statistics ---
     # These stats will be used in the daily ranking, so we make the bucket < 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS sender_stats_1h CASCADE;
-        
+
         CREATE VIEW sender_stats_1h
         WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='5 minutes') AS
         SELECT
@@ -90,7 +92,7 @@ def create_timescaledb_views():
         GROUP BY bucket, sp.name, is_trustworthy;
     """)
 
-    ### Receiver statistics
+    # --- Receiver statistics ---
     # These stats will be used in the daily ranking, so we make the bucket < 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS receiver_stats_1h CASCADE;
@@ -128,8 +130,8 @@ def create_timescaledb_views():
         FROM sender_positions AS sp
         GROUP BY bucket, sp.receiver_name, is_trustworthy;
     """)
-    
+
-    ### Relation statistics (sender <-> receiver)
+    # --- Relation statistics (sender <-> receiver) ---
     # these stats will be used on a >= 1d basis, so we make the bucket = 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS relation_stats_1d CASCADE;
@@ -162,4 +164,4 @@ class MyView(db.Model):
         autoload=True,
         autoload_with=db.engine
     )
-"""
+"""

app/commands/database.py

@@ -119,6 +119,7 @@ def import_airports(path="tests/SeeYou.cup"):
     db.session.commit()
     print("Imported {} airports.".format(len(airports)))
 
+
 @user_cli.command("create_timescaledb_views")
 def cmd_create_timescaledb_views():
     """Create TimescaleDB views."""
@@ -126,11 +127,10 @@ def cmd_create_timescaledb_views():
     create_timescaledb_views()
     print("Done")
 
+
 @user_cli.command("create_views")
 def cmd_create_views():
     """Create views."""
 
     create_views()
     print("Done")
-
-

app/commands/export.py

@@ -15,6 +15,7 @@ from app import db
 user_cli = AppGroup("export")
 user_cli.help = "Export data in several file formats."
 
+
 @user_cli.command("debug_sql")
 @click.argument("start")
 @click.argument("end")
@@ -24,7 +25,7 @@ def debug_sql(start, end, name):
 
     # First: get all the positions (and the receiver names for later)
     sql_sender_positions = f"""
-        SELECT reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl 
+        SELECT reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl
         FROM sender_positions
         WHERE reference_timestamp BETWEEN '{start}' AND '{end}' AND name = '{name}'
         ORDER BY reference_timestamp;
@@ -38,7 +39,7 @@ def debug_sql(start, end, name):
             receiver_names.append("'" + row[2] + "'")
         row = [f"'{r}'" if r else "DEFAULT" for r in row]
         sender_position_values.append(f"({','.join(row)})")
-    
+
     # Second: get the receivers
     sql_receivers = f"""
         SELECT name, location
@@ -50,8 +51,8 @@ def debug_sql(start, end, name):
     results = db.session.execute(sql_receivers)
     for row in results:
         row = [f"'{r}'" if r else "DEFAULT" for r in row]
-        receiver_values.append(f"({','.join(row)})")    
+        receiver_values.append(f"({','.join(row)})")
-    
+
     # Third: get the airports
     sql_airports = f"""
         SELECT DISTINCT a.name, a.location, a.altitude, a.style, a.border
@@ -66,23 +67,22 @@ def debug_sql(start, end, name):
     results = db.session.execute(sql_airports)
     for row in results:
         row = [f"'{r}'" if r else "DEFAULT" for r in row]
-        airport_values.append(f"({','.join(row)})")  
+        airport_values.append(f"({','.join(row)})")
 
     # Last: write all into file
     with open(f'{start}_{end}_{name}.sql', 'w') as file:
-        file.write(f'/*\n')
+        file.write('/*\n')
-        file.write(f'OGN Python SQL Export\n')
+        file.write('OGN Python SQL Export\n')
         file.write(f'Created by: {os.getlogin()}\n')
         file.write(f'Created at: {datetime.datetime.utcnow()}\n')
-        file.write(f'*/\n\n')
+        file.write('*/\n\n')
-
 
         file.write("INSERT INTO airports(name, location, altitude, style, border) VALUES\n")
         file.write(',\n'.join(airport_values) + ';\n\n')
 
         file.write("INSERT INTO receivers(name, location) VALUES\n")
         file.write(',\n'.join(receiver_values) + ';\n\n')
-        
+
         file.write("INSERT INTO sender_positions(reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl) VALUES\n")
         file.write(',\n'.join(sender_position_values) + ';\n\n')
 
@@ -139,7 +139,7 @@ def igc(address, date):
         return
 
     try:
-        sender = db.session.query(Sender).filter(Sender.address==address).one()
+        sender = db.session.query(Sender).filter(Sender.address == address).one()
     except NoResultFound as e:
         print(f"No data for '{address}' in the DB")
         return
@@ -173,9 +173,9 @@ def igc(address, date):
 
         points = (
             db.session.query(SenderPosition)
-                .filter(db.between(SenderPosition.reference_timestamp, f"{date} 00:00:00", f"{date} 23:59:59"))
+            .filter(db.between(SenderPosition.reference_timestamp, f"{date} 00:00:00", f"{date} 23:59:59"))
-                .filter(SenderPosition.name == sender.name)
+            .filter(SenderPosition.name == sender.name)
-                .order_by(SenderPosition.timestamp)
+            .order_by(SenderPosition.timestamp)
         )
 
         for point in points:

app/commands/gateway.py

@@ -84,7 +84,7 @@ def transfer():
     """Transfer data from redis to the database."""
 
     transfer_from_redis_to_database()
-       
+
 
 @user_cli.command("printout")
 @click.option("--aprs_filter", default='')
@@ -101,36 +101,3 @@ def printout(aprs_filter):
         current_app.logger.warning("\nStop ogn gateway")
 
     client.disconnect()
-
-@user_cli.command("convert")
-@click.argument("path")
-def file_import(path):
-    """Convert APRS logfiles into csv files for fast bulk import."""
-
-    for (root, dirs, files) in os.walk(path):
-        for file in sorted(files):
-            print(file)
-            convert(os.path.join(root, file))
-
-
-@user_cli.command("calculate")
-@click.argument("path")
-def file_calculate(path):
-    """Import csv files, calculate geographic features (distance, radial, agl, ...) and make data distinct."""
-
-    file_tuples = []
-    for (root, dirs, files) in os.walk(path):
-        for file in sorted(files):
-            if file.startswith('aircraft_beacons') and file.endswith('.csv.gz'):
-                ab_filename = os.path.join(root, file)
-                rb_filename = os.path.join(root, 'receiver' + file[8:])
-                target_filename = os.path.join(root, file + '2')
-                if os.path.isfile(target_filename):
-                    print("Outputfile {} already exists. Skipping".format(target_filename))
-                else:
-                    file_tuples.append((ab_filename, rb_filename, target_filename))
-
-    pbar = tqdm(file_tuples)
-    for file_tuple in pbar:
-        pbar.set_description("Converting {}".format(file_tuple[0]))
-        calculate(file_tuple[0], file_tuple[1], file_tuple[2])

app/gateway/beacon_conversion.py

@@ -40,7 +40,6 @@ def aprs_string_to_message(aprs_string):
             bearing = int(message['bearing'])
             message['bearing'] = bearing if bearing < 360 else 0
 
-
         if "aircraft_type" in message:
             message["aircraft_type"] = AircraftType(message["aircraft_type"]) if message["aircraft_type"] in AircraftType.list() else AircraftType.UNKNOWN
 

app/gateway/message_handling.py

@@ -18,11 +18,11 @@ SENDER_POSITION_BEACON_FIELDS = [
     "receiver_name",
     "timestamp",
     "location",
-    
+
     "track",
     "ground_speed",
     "altitude",
-    
+
     "address_type",
     "aircraft_type",
     "stealth",
@@ -38,7 +38,7 @@ SENDER_POSITION_BEACON_FIELDS = [
     "hardware_version",
     "real_address",
     "signal_power",
-    
+
     "distance",
     "bearing",
     "normalized_quality",
@@ -66,7 +66,7 @@ RECEIVER_POSITION_BEACON_FIELDS = [
 
 RECEIVER_STATUS_BEACON_FIELDS = [
     "reference_timestamp",
-    
+
     "name",
     "dstcall",
     "receiver_name",
@@ -90,7 +90,7 @@ def sender_position_message_to_csv_string(message, none_character=''):
 
     csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23},{24},{25},{26},{27},{28},{29},{30}\n".format(
         message['reference_timestamp'],
-        
+
         message['name'],
         message['dstcall'],
         message['relay'] if 'relay' in message and message['relay'] else none_character,
@@ -101,8 +101,8 @@ def sender_position_message_to_csv_string(message, none_character=''):
         message['track'] if 'track' in message and message['track'] else none_character,
         message['ground_speed'] if 'ground_speed' in message and message['ground_speed'] else none_character,
         int(message['altitude']) if message['altitude'] else none_character,
-       
+
-        message['address_type'] if 'address_type' in message and message['address_type'] else none_character,   #10
+        message['address_type'] if 'address_type' in message and message['address_type'] else none_character,   # 10
         message['aircraft_type'].name if 'aircraft_type' in message and message['aircraft_type'] else AircraftType.UNKNOWN.name,
         message['stealth'] if 'stealth' in message and message['stealth'] else none_character,
         message['address'] if 'address' in message and message['address'] else none_character,
@@ -112,12 +112,12 @@ def sender_position_message_to_csv_string(message, none_character=''):
         message['error_count'] if 'error_count' in message and message['error_count'] else none_character,
         message['frequency_offset'] if 'frequency_offset' in message and message['frequency_offset'] else none_character,
         message['gps_quality_horizontal'] if 'gps_quality_horizontal' in message and message['gps_quality_horizontal'] else none_character,
-        message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character, #20
+        message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character,   # 20
         message['software_version'] if 'software_version' in message and message['software_version'] else none_character,
         message['hardware_version'] if 'hardware_version' in message and message['hardware_version'] else none_character,
         message['real_address'] if 'real_address' in message and message['real_address'] else none_character,
         message['signal_power'] if 'signal_power' in message and message['signal_power'] else none_character,
-        
+
         message['distance'] if 'distance' in message and message['distance'] else none_character,
         message['bearing'] if 'bearing' in message and message['bearing'] else none_character,
         message['normalized_quality'] if 'normalized_quality' in message and message['normalized_quality'] else none_character,
@@ -132,7 +132,7 @@ def sender_position_message_to_csv_string(message, none_character=''):
 def receiver_position_message_to_csv_string(message, none_character=''):
     csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n".format(
         message['reference_timestamp'],
-        
+
         message['name'],
         message['dstcall'],
         message['receiver_name'],
@@ -180,7 +180,7 @@ def sender_position_csv_strings_to_db(lines):
 
     cursor.execute(f"CREATE TEMPORARY TABLE {tmp_tablename} (LIKE sender_positions) ON COMMIT DROP;")
     cursor.copy_from(file=string_buffer, table=tmp_tablename, sep=",", columns=SENDER_POSITION_BEACON_FIELDS)
-    
+
     # Update agl
     cursor.execute(f"""
         UPDATE {tmp_tablename} AS tmp
@@ -238,7 +238,7 @@ def sender_position_csv_strings_to_db(lines):
     """)
 
     # Update sender_infos FK -> senders
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE sender_infos AS si
         SET sender_id = s.id
         FROM senders AS s
@@ -316,7 +316,7 @@ def receiver_position_csv_strings_to_db(lines):
             tmp.name,
             tmp.timestamp,
             tmp.location,
-            
+
             tmp.altitude,
 
             tmp.agl
@@ -340,18 +340,18 @@ def receiver_position_csv_strings_to_db(lines):
     """)
 
     # Update receiver country
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE receivers AS r
-        SET 
+        SET
             country_id = c.gid
         FROM countries AS c
         WHERE r.country_id IS NULL AND ST_Within(r.location, c.geom);
     """)
 
     # Update receiver airport
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE receivers AS r
-        SET 
+        SET
             airport_id = (
                 SELECT id
                 FROM airports AS a
@@ -400,7 +400,7 @@ def receiver_status_csv_strings_to_db(lines):
 
             tmp.name,
             tmp.timestamp,
-            
+
             tmp.version,
             tmp.platform,
 

app/gateway/process_tools.py

@@ -37,6 +37,7 @@ class Timer(object):
             print("[{}]".format(self.name))
         print("Elapsed: {}".format(time.time() - self.tstart))
 
+
 def export_to_path(path):
     connection = db.engine.raw_connection()
     cursor = connection.cursor()

app/main/__init__.py

@@ -3,4 +3,4 @@ from flask import Blueprint
 bp = Blueprint("main", __name__)
 
 import app.main.routes
-import app.main.jinja_filters
+import app.main.jinja_filters

app/main/jinja_filters.py

@@ -6,6 +6,7 @@ import time
 import datetime
 import math
 
+
 @bp.app_template_filter()
 def timestamp_to_status(timestamp):
     if datetime.datetime.utcnow() - timestamp < datetime.timedelta(minutes=10):
@@ -15,6 +16,7 @@ def timestamp_to_status(timestamp):
     else:
         return '<b>OFFLINE</b>'
 
+
 @bp.app_template_filter()
 def to_html_link(obj):
     if isinstance(obj, Airport):
@@ -40,6 +42,7 @@ def to_html_link(obj):
     else:
         raise NotImplementedError("cant apply filter 'to_html_link' to object {type(obj)}")
 
+
 @bp.app_template_filter()
 def to_ordinal(rad):
     deg = math.degrees(rad)
@@ -58,4 +61,4 @@ def to_ordinal(rad):
     elif deg >= 247.5 and deg < 292.5:
         return "E"
     elif deg >= 292.5 and deg < 337.5:
-        return "NE"
+        return "NE"

app/main/matplotlib_service.py

@@ -1,10 +1,11 @@
 from app import db
-from app.model import *
+from app.model import SenderDirectionStatistic
 import random
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib.figure import Figure
 
+
 def create_range_figure2(sender_id):
     fig = Figure()
     axis = fig.add_subplot(1, 1, 1)
@@ -14,6 +15,7 @@ def create_range_figure2(sender_id):
 
     return fig
 
+
 def create_range_figure(sender_id):
     sds = db.session.query(SenderDirectionStatistic) \
         .filter(SenderDirectionStatistic.sender_id == sender_id) \
@@ -24,11 +26,11 @@ def create_range_figure(sender_id):
     fig = Figure()
 
     direction_data = sds.direction_data
-    max_range = max([r['max_range']/1000.0 for r in direction_data])
+    max_range = max([r['max_range'] / 1000.0 for r in direction_data])
 
-    theta = np.array([i['direction']/180*np.pi for i in direction_data])
+    theta = np.array([i['direction'] / 180 * np.pi for i in direction_data])
-    radii = np.array([i['max_range']/1000 if i['max_range'] > 0 else 0 for i in direction_data])
+    radii = np.array([i['max_range'] / 1000 if i['max_range'] > 0 else 0 for i in direction_data])
-    width = np.array([13/180*np.pi for i in direction_data])
+    width = np.array([13 / 180 * np.pi for i in direction_data])
     colors = plt.cm.viridis(radii / max_range)
 
     ax = fig.add_subplot(111, projection='polar')
@@ -38,5 +40,5 @@ def create_range_figure(sender_id):
     ax.set_theta_direction(-1)
 
     fig.suptitle(f"Range between sender '{sds.sender.name}' and receiver '{sds.receiver.name}'")
-    
+
-    return fig
+    return fig

app/main/routes.py

@@ -25,7 +25,7 @@ def get_used_countries():
 
 @cache.memoize()
 def get_used_airports_by_country(sel_country):
-    query = db.session.query(Airport).filter(Airport.country_code == sel_country).filter(TakeoffLanding.airport_id==Airport.id).filter(TakeoffLanding.country_id == Country.gid).order_by(Airport.name).distinct(Airport.name)
+    query = db.session.query(Airport).filter(Airport.country_code == sel_country).filter(TakeoffLanding.airport_id == Airport.id).filter(TakeoffLanding.country_id == Country.gid).order_by(Airport.name).distinct(Airport.name)
     return [used_airport for used_airport in query]
 
 
@@ -45,17 +45,18 @@ def get_dates_for_airport(sel_airport):
 @bp.route("/")
 @bp.route("/index.html")
 def index():
-    today_beginning = datetime.combine(date.today(), time()) 
+    today_beginning = datetime.combine(date.today(), time())
-    
+
-    senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen>=today_beginning).one()[0]
+    senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen >= today_beginning).one()[0]
-    receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen>=today_beginning).one()[0]
+    receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen >= today_beginning).one()[0]
-    takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==True)).one()[0]
+    takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is True)).one()[0]
-    landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==False)).one()[0]
+    landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is False)).one()[0]
-    sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date==date.today()).one()[0]
+    sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date == date.today()).one()[0]
     sender_positions_total = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).one()[0]
 
     last_logbook_entries = db.session.query(Logbook).order_by(Logbook.reference_timestamp.desc()).limit(10)
-    return render_template("index.html",
+    return render_template(
+        "index.html",
         senders_today=senders_today,
         receivers_today=receivers_today,
         takeoffs_today=takeoffs_today,
@@ -80,13 +81,14 @@ def sender_detail():
 
     return render_template("sender_detail.html", title="Sender", sender=sender)
 
+
 @bp.route("/range_view.png")
 def range_view():
     import io
     from flask import Response
 
     from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
-    
+
     sender_id = request.args.get("sender_id")
 
     fig = create_range_figure(sender_id)
@@ -210,24 +212,28 @@ def download_flight():
 
     return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain")
 
+
 @bp.route("/sender_ranking.html")
 def sender_ranking():
     sender_statistics = db.session.query(SenderStatistic) \
-        .filter(db.and_(SenderStatistic.date==date.today(), SenderStatistic.is_trustworthy==True)) \
+        .filter(db.and_(SenderStatistic.date == date.today(), SenderStatistic.is_trustworthy is True)) \
         .order_by(SenderStatistic.max_distance.desc()) \
         .all()
 
-    return render_template("sender_ranking.html",
+    return render_template(
+        "sender_ranking.html",
         title="Sender Ranking",
         ranking=sender_statistics)
 
+
 @bp.route("/receiver_ranking.html")
 def receiver_ranking():
     receiver_statistics = db.session.query(ReceiverStatistic) \
-        .filter(db.and_(ReceiverStatistic.date==date.today(), ReceiverStatistic.is_trustworthy==True)) \
+        .filter(db.and_(ReceiverStatistic.date == date.today(), ReceiverStatistic.is_trustworthy is True)) \
         .order_by(ReceiverStatistic.max_distance.desc()) \
         .all()
 
-    return render_template("receiver_ranking.html",
+    return render_template(
+        "receiver_ranking.html",
         title="Receiver Ranking",
         ranking=receiver_statistics)

app/model/logbook.py

@@ -37,7 +37,7 @@ class Logbook(db.Model):
     @duration.expression
     def duration(cls):
         return db.case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != db.null() and cls.takeoff_timestamp != db.null())
-    
+
     @hybrid_property
     def reference_timestamp(self):
         return self.takeoff_timestamp if self.takeoff_timestamp is not None else self.landing_timestamp
@@ -50,7 +50,7 @@ class Logbook(db.Model):
     # FIXME: does not work...
 
 # FIXME: this does not throw an error as the __table_args__ above, but there is no index created
-#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})"   
+#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})"
 #Index("idx_logbook_reference_timestamp", _wrapped_case)
 
 # TODO:

app/model/receiver.py

@@ -47,11 +47,10 @@ class Receiver(db.Model):
     def airports_nearby(self):
         query = (
             db.session.query(Airport, db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt), db.func.st_azimuth(self.location_wkt, Airport.location_wkt))
-                .filter(db.func.st_contains(db.func.st_buffer(Airport.location_wkt, 1), self.location_wkt))
+            .filter(db.func.st_contains(db.func.st_buffer(Airport.location_wkt, 1), self.location_wkt))
-                .filter(Airport.style.in_((2,4,5)))
+            .filter(Airport.style.in_((2, 4, 5)))
-                .order_by(db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt).asc())
+            .order_by(db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt).asc())
-                .limit(5)
+            .limit(5)
         )
-        airports = [(airport,distance,azimuth) for airport, distance, azimuth in query]
+        airports = [(airport, distance, azimuth) for airport, distance, azimuth in query]
         return airports
-

app/model/sender_direction_statistic.py

@@ -2,6 +2,7 @@ from app import db
 
 from sqlalchemy.dialects.postgresql import JSON
 
+
 class SenderDirectionStatistic(db.Model):
     __tablename__ = "sender_direction_statistics"
 

app/model/sender_info.py

@@ -4,6 +4,7 @@ from .aircraft_type import AircraftType
 
 #from sqlalchemy.dialects.postgresql import ENUM
 
+
 class SenderInfo(db.Model):
     __tablename__ = "sender_infos"
 

app/model/sender_position.py

@@ -48,7 +48,7 @@ class SenderPosition(db.Model):
     hardware_version = db.Column(db.SmallInteger)
     real_address = db.Column(db.String(6))
     signal_power = db.Column(db.Float(precision=2))
-    
+
     #proximity = None
 
     # Calculated values (from parser)
@@ -60,4 +60,3 @@ class SenderPosition(db.Model):
     location_mgrs = db.Column(db.String(15))                # full mgrs (15 chars)
     location_mgrs_short = db.Column(db.String(9))           # reduced mgrs (9 chars), e.g. used for melissas range tool
     agl = db.Column(db.Float(precision=2))
-

app/model/sender_position_statistic.py

@@ -4,6 +4,7 @@ from .aircraft_type import AircraftType
 
 from sqlalchemy.dialects.postgresql import ENUM
 
+
 class SenderPositionStatistic(db.Model):
     __tablename__ = "sender_position_statistics"
 

app/tasks/orm_tasks.py

@@ -9,13 +9,14 @@ from app.collect.gateway import transfer_from_redis_to_database
 
 from app import db, celery
 
+
 @celery.task(name="transfer_to_database")
 def transfer_to_database():
     """Transfer APRS data from Redis to database."""
-    
+
     result = transfer_from_redis_to_database()
     return result
-    
+
 
 @celery.task(name="update_takeoff_landings")
 def update_takeoff_landings(last_minutes):

app/tasks/sql_tasks.py

@@ -10,7 +10,7 @@ def update_statistics(date_str=None):
     if date_str is None:
         date_str = datetime.utcnow().strftime("%Y-%m-%d")
 
-    # Update relation statistics 
+    # Update relation statistics
     db.session.execute(f"""
         DELETE FROM relation_statistics
         WHERE date = '{date_str}';
@@ -22,7 +22,7 @@ def update_statistics(date_str=None):
             tmp.receiver_id,
 
             is_trustworthy,
-            
+
             MAX(tmp.max_distance) AS max_distance,
             MAX(tmp.max_normalized_quality) AS max_normalized_quality,
             SUM(tmp.messages_count) AS messages_count,
@@ -43,7 +43,7 @@ def update_statistics(date_str=None):
             tmp.sender_id,
 
             is_trustworthy,
-            
+
             MAX(tmp.max_distance) AS max_distance,
             MAX(tmp.max_normalized_quality) AS max_normalized_quality,
             SUM(tmp.messages_count) AS messages_count,
@@ -65,7 +65,7 @@ def update_statistics(date_str=None):
             tmp.receiver_id,
 
             is_trustworthy,
-            
+
             MAX(tmp.max_distance) AS max_distance,
             MAX(tmp.max_normalized_quality) AS max_normalized_quality,
             SUM(tmp.messages_count) AS messages_count,
@@ -84,7 +84,7 @@ def update_sender_direction_statistics():
     """ Update sender_direction_statistics."""
 
     db.session.execute("""
-		DELETE FROM sender_direction_statistics;
+        DELETE FROM sender_direction_statistics;
 
         INSERT INTO sender_direction_statistics(sender_id, receiver_id, directions_count, messages_count, direction_data)
         SELECT
@@ -93,7 +93,7 @@ def update_sender_direction_statistics():
             COUNT(sq2.*) AS directions_count,
             SUM(sq2.messages_count) AS messages_count,
             json_agg(json_build_object('direction', direction, 'messages_count', messages_count, 'max_range', max_range)) AS direction_data
-		FROM (
+        FROM (
             SELECT
                 sq.sender_id,
                 sq.receiver_id,

app/utils.py

@@ -137,8 +137,9 @@ def open_file(filename):
         f = open(filename, "rt", encoding="latin-1")
         return f
 
+
 def get_sql_trustworthy(source_table_alias):
-    MIN_DISTANCE =   1000
+    MIN_DISTANCE = 1000
     MAX_DISTANCE = 640000
     MAX_NORMALIZED_QUALITY = 40     # this is enough for > 640km
     MAX_ERROR_COUNT = 5
@@ -149,4 +150,4 @@ def get_sql_trustworthy(source_table_alias):
         AND ({source_table_alias}.normalized_quality IS NOT NULL AND {source_table_alias}.normalized_quality < {MAX_NORMALIZED_QUALITY})
         AND ({source_table_alias}.error_count IS NULL OR {source_table_alias}.error_count < {MAX_ERROR_COUNT})
         AND ({source_table_alias}.climb_rate IS NULL OR {source_table_alias}.climb_rate BETWEEN -{MAX_CLIMB_RATE} AND {MAX_CLIMB_RATE})
-    """
+    """

config.py

@@ -1,25 +1,27 @@
 import os
 
+
 class BaseConfig:
     SECRET_KEY = "i-like-ogn"
-    
+
     # Flask-Cache stuff
     CACHE_TYPE = "simple"
     CACHE_DEFAULT_TIMEOUT = 300
-    
+
     # Redis stuff
     REDIS_URL = "redis://localhost:6379/0"
-    
+
     # Celery stuff
     BROKER_URL = os.environ.get("BROKER_URL", REDIS_URL)
     CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", REDIS_URL)
 
     APRS_USER = "OGNPYTHON"
 
+
 class DefaultConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "postgresql://postgres:postgres@localhost:5432/ogn")
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    
+
     # Celery beat stuff
     from celery.schedules import crontab
     from datetime import timedelta
@@ -33,17 +35,19 @@ class DefaultConfig(BaseConfig):
 
         "update_ddb_daily": {"task": "import_ddb", "schedule": timedelta(days=1)},
         #"update_logbook_max_altitude": {"task": "update_logbook_max_altitude", "schedule": timedelta(minutes=1), "kwargs": {"offset_days": 0}},
-        
+
         #"purge_old_data": {"task": "purge_old_data", "schedule": timedelta(hours=1), "kwargs": {"max_hours": 48}},
     }
 
+
 class DevelopmentConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5432/ogn_test"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SQLALCHEMY_ECHO = False
 
+
 configs = {
     'default': DefaultConfig,
     'development': DevelopmentConfig,
     'testing': DevelopmentConfig
-}
+}

setup.cfg

@@ -1,2 +1,2 @@
 [flake8]
-ignore = F401, F841, E402, E501, E126
+ignore = F401, F841, E402, E501, E126, E265

tests/collect/test_logbook.py

@@ -76,7 +76,6 @@ class TestLogbook(TestBaseDB):
         self.assertEqual(entries[0].takeoff_airport_id, self.koenigsdorf.id)
         self.assertEqual(entries[0].landing_airport_id, self.koenigsdorf.id)
 
-
     @unittest.skip('needs information about airport timezone')
     def test_takeoff_and_landing_on_different_days(self):
         db.session.add(self.takeoff_koenigsdorf_dd0815)