PEP8 violations

app/__init__.py

@@ -39,10 +39,12 @@ def create_app(config_name='default'):
 
     return app
 
+
 def register_blueprints(app):
     from app.main import bp as bp_main
     app.register_blueprint(bp_main)
 
+
 def init_celery(app=None):
     app = app or create_app(os.getenv('FLASK_CONFIG') or 'default')
     celery.conf.broker_url = app.config['BROKER_URL']

app/collect/flights.py

@@ -6,6 +6,7 @@ NOTHING = ""
 CONTEST_RELEVANT = "AND agl < 1000"
 LOW_PASS = "AND agl < 50 and ground_speed > 250"
 
+
 def compute_flights(date, flight_type=0):
     if flight_type == 0:
         filter = NOTHING
@@ -66,6 +67,7 @@ def compute_flights(date, flight_type=0):
     db.session.execute(query)
     db.session.commit()
 
+
 def compute_gaps(date):
     date_str = date.strftime("%Y-%m-%d")
 
@@ -105,6 +107,7 @@ def compute_gaps(date):
     db.session.execute(query)
     db.session.commit()
 
+
 if __name__ == '__main__':
     from app import create_app
     app = create_app()

app/collect/gateway.py

@@ -4,8 +4,10 @@ from flask import current_app
 from app import redis_client
 from app.gateway.message_handling import sender_position_csv_strings_to_db, receiver_position_csv_strings_to_db, receiver_status_csv_strings_to_db
 
+
 def transfer_from_redis_to_database():
-    unmapping = lambda s: s[0].decode('utf-8')
+    def unmapping(string):
+        return string[0].decode('utf-8')
 
     receiver_status_data = list(map(unmapping, redis_client.zpopmin('receiver_status', 100000)))
     receiver_position_data = list(map(unmapping, redis_client.zpopmin('receiver_position', 100000)))

app/collect/logbook.py

@@ -181,7 +181,6 @@ def update_logbook(offset_days=None):
             db.func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"),
             db.func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next")
         )
-        #.filter(between(TakeoffLanding.timestamp, start, end))
         .subquery()
     )
 
@@ -253,7 +252,7 @@ def update_logbook(offset_days=None):
 
     # insert (new) flights
     new_flights_query = (
-        db.session.query(complete_flight_query) \
+        db.session.query(complete_flight_query)
         .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.landing_timestamp == complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id == complete_flight_query.c.landing_airport_id)).exists())
         .filter(~Logbook.query.filter(db.and_(Logbook.sender_id == complete_flight_query.c.sender_id, Logbook.takeoff_timestamp == complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id == complete_flight_query.c.takeoff_airport_id)).exists())
     )
@@ -285,8 +284,7 @@ def update_logbook(offset_days=None):
         )) \
         .values(takeoff_timestamp=complete_flight_query.c.takeoff_timestamp,
                 takeoff_track=complete_flight_query.c.takeoff_track,
-                takeoff_airport_id=complete_flight_query.c.takeoff_airport_id
-        )
+                takeoff_airport_id=complete_flight_query.c.takeoff_airport_id)
     result = db.session.execute(upd)
     current_app.logger.debug(f"Updated {result.rowcount} takeoffs to complete flights")
     db.session.commit()
@@ -303,8 +301,7 @@ def update_logbook(offset_days=None):
         )) \
         .values(landing_timestamp=complete_flight_query.c.landing_timestamp,
                 landing_track=complete_flight_query.c.landing_track,
-                landing_airport_id=complete_flight_query.c.landing_airport_id
-        )
+                landing_airport_id=complete_flight_query.c.landing_airport_id)
     result = db.session.execute(upd)
     current_app.logger.debug(f"Updated {result.rowcount} landings to complete flights")
     db.session.commit()
@@ -312,11 +309,10 @@ def update_logbook(offset_days=None):
     return
 
 
-
 def update_max_altitudes():
     MAX_UPDATES = 60
 
-    query = f"""
+    query = """
         UPDATE logbooks
             SET max_altitude = sq2.max_altitude
         FROM (
@@ -347,6 +343,7 @@ def update_max_altitudes():
 
     return update_counter
 
+
 def update_max_altitudes_orm():
     """Add max altitudes in logbook when flight is complete (takeoff and landing)."""
 
@@ -374,11 +371,12 @@ def update_max_altitudes_orm():
     finish_message = "Logbook (altitude): {} entries updated.".format(update_logbooks)
     return finish_message
 
+
 if __name__ == '__main__':
     from app import create_app
     app = create_app()
     with app.app_context():
-        #result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0))
-        #result = update_logbook()
+        result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0))
+        result = update_logbook()
         result = update_max_altitudes_orm()
         print(result)

app/collect/timescaledb_views.py

@@ -3,8 +3,9 @@ from app.utils import get_sql_trustworthy
 
 SQL_TRUSTWORTHY = get_sql_trustworthy(source_table_alias='sp')
 
+
 def create_views():
-    db.session.execute(f"""
+    db.session.execute("""
         DROP VIEW IF EXISTS receiver_ranking CASCADE;
 
         CREATE VIEW receiver_ranking AS
@@ -23,7 +24,7 @@ def create_views():
         ORDER BY max_distance DESC;
     """)
 
-    db.session.execute(f"""
+    db.session.execute("""
         DROP VIEW IF EXISTS sender_ranking CASCADE;
 
         CREATE VIEW sender_ranking AS
@@ -44,6 +45,7 @@ def create_views():
 
     db.session.commit()
 
+
 def create_timescaledb_views():
     # 1. Since the reference_timestamps are strictly increasing we can set
     #    the parameter 'refresh_lag' to a very short time so the materialization
@@ -51,7 +53,7 @@ def create_timescaledb_views():
     # 2. The feature realtime aggregation from TimescaleDB is quite time consuming.
     #    So we set materialized_only=true
 
-    ### Sender statistics
+    # --- Sender statistics ---
     # These stats will be used in the daily ranking, so we make the bucket < 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS sender_stats_1h CASCADE;
@@ -90,7 +92,7 @@ def create_timescaledb_views():
         GROUP BY bucket, sp.name, is_trustworthy;
     """)
 
-    ### Receiver statistics
+    # --- Receiver statistics ---
     # These stats will be used in the daily ranking, so we make the bucket < 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS receiver_stats_1h CASCADE;
@@ -129,7 +131,7 @@ def create_timescaledb_views():
         GROUP BY bucket, sp.receiver_name, is_trustworthy;
     """)
 
-    ### Relation statistics (sender <-> receiver)
+    # --- Relation statistics (sender <-> receiver) ---
     # these stats will be used on a >= 1d basis, so we make the bucket = 1d
     db.session.execute(f"""
         DROP VIEW IF EXISTS relation_stats_1d CASCADE;

app/commands/database.py

@@ -119,6 +119,7 @@ def import_airports(path="tests/SeeYou.cup"):
     db.session.commit()
     print("Imported {} airports.".format(len(airports)))
 
+
 @user_cli.command("create_timescaledb_views")
 def cmd_create_timescaledb_views():
     """Create TimescaleDB views."""
@@ -126,11 +127,10 @@ def cmd_create_timescaledb_views():
     create_timescaledb_views()
     print("Done")
 
+
 @user_cli.command("create_views")
 def cmd_create_views():
     """Create views."""
 
     create_views()
     print("Done")
-
-

app/commands/export.py

@@ -15,6 +15,7 @@ from app import db
 user_cli = AppGroup("export")
 user_cli.help = "Export data in several file formats."
 
+
 @user_cli.command("debug_sql")
 @click.argument("start")
 @click.argument("end")
@@ -70,12 +71,11 @@ def debug_sql(start, end, name):
 
     # Last: write all into file
     with open(f'{start}_{end}_{name}.sql', 'w') as file:
-        file.write(f'/*\n')
-        file.write(f'OGN Python SQL Export\n')
+        file.write('/*\n')
+        file.write('OGN Python SQL Export\n')
         file.write(f'Created by: {os.getlogin()}\n')
         file.write(f'Created at: {datetime.datetime.utcnow()}\n')
-        file.write(f'*/\n\n')
-
+        file.write('*/\n\n')
 
         file.write("INSERT INTO airports(name, location, altitude, style, border) VALUES\n")
         file.write(',\n'.join(airport_values) + ';\n\n')

app/commands/gateway.py

@@ -101,36 +101,3 @@ def printout(aprs_filter):
         current_app.logger.warning("\nStop ogn gateway")
 
     client.disconnect()
-
-@user_cli.command("convert")
-@click.argument("path")
-def file_import(path):
-    """Convert APRS logfiles into csv files for fast bulk import."""
-
-    for (root, dirs, files) in os.walk(path):
-        for file in sorted(files):
-            print(file)
-            convert(os.path.join(root, file))
-
-
-@user_cli.command("calculate")
-@click.argument("path")
-def file_calculate(path):
-    """Import csv files, calculate geographic features (distance, radial, agl, ...) and make data distinct."""
-
-    file_tuples = []
-    for (root, dirs, files) in os.walk(path):
-        for file in sorted(files):
-            if file.startswith('aircraft_beacons') and file.endswith('.csv.gz'):
-                ab_filename = os.path.join(root, file)
-                rb_filename = os.path.join(root, 'receiver' + file[8:])
-                target_filename = os.path.join(root, file + '2')
-                if os.path.isfile(target_filename):
-                    print("Outputfile {} already exists. Skipping".format(target_filename))
-                else:
-                    file_tuples.append((ab_filename, rb_filename, target_filename))
-
-    pbar = tqdm(file_tuples)
-    for file_tuple in pbar:
-        pbar.set_description("Converting {}".format(file_tuple[0]))
-        calculate(file_tuple[0], file_tuple[1], file_tuple[2])

app/gateway/beacon_conversion.py

@@ -40,7 +40,6 @@ def aprs_string_to_message(aprs_string):
             bearing = int(message['bearing'])
             message['bearing'] = bearing if bearing < 360 else 0
 
-
         if "aircraft_type" in message:
             message["aircraft_type"] = AircraftType(message["aircraft_type"]) if message["aircraft_type"] in AircraftType.list() else AircraftType.UNKNOWN
 

app/gateway/message_handling.py

@@ -238,7 +238,7 @@ def sender_position_csv_strings_to_db(lines):
     """)
 
     # Update sender_infos FK -> senders
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE sender_infos AS si
         SET sender_id = s.id
         FROM senders AS s
@@ -340,7 +340,7 @@ def receiver_position_csv_strings_to_db(lines):
     """)
 
     # Update receiver country
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE receivers AS r
         SET
             country_id = c.gid
@@ -349,7 +349,7 @@ def receiver_position_csv_strings_to_db(lines):
     """)
 
     # Update receiver airport
-    cursor.execute(f"""
+    cursor.execute("""
         UPDATE receivers AS r
         SET
             airport_id = (

app/gateway/process_tools.py

@@ -37,6 +37,7 @@ class Timer(object):
             print("[{}]".format(self.name))
         print("Elapsed: {}".format(time.time() - self.tstart))
 
+
 def export_to_path(path):
     connection = db.engine.raw_connection()
     cursor = connection.cursor()

app/main/jinja_filters.py

@@ -6,6 +6,7 @@ import time
 import datetime
 import math
 
+
 @bp.app_template_filter()
 def timestamp_to_status(timestamp):
     if datetime.datetime.utcnow() - timestamp < datetime.timedelta(minutes=10):
@@ -15,6 +16,7 @@ def timestamp_to_status(timestamp):
     else:
         return '<b>OFFLINE</b>'
 
+
 @bp.app_template_filter()
 def to_html_link(obj):
     if isinstance(obj, Airport):
@@ -40,6 +42,7 @@ def to_html_link(obj):
     else:
         raise NotImplementedError("cant apply filter 'to_html_link' to object {type(obj)}")
 
+
 @bp.app_template_filter()
 def to_ordinal(rad):
     deg = math.degrees(rad)

app/main/matplotlib_service.py

@@ -1,10 +1,11 @@
 from app import db
-from app.model import *
+from app.model import SenderDirectionStatistic
 import random
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib.figure import Figure
 
+
 def create_range_figure2(sender_id):
     fig = Figure()
     axis = fig.add_subplot(1, 1, 1)
@@ -14,6 +15,7 @@ def create_range_figure2(sender_id):
 
     return fig
 
+
 def create_range_figure(sender_id):
     sds = db.session.query(SenderDirectionStatistic) \
         .filter(SenderDirectionStatistic.sender_id == sender_id) \

app/main/routes.py

@@ -49,13 +49,14 @@ def index():
 
     senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen >= today_beginning).one()[0]
     receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen >= today_beginning).one()[0]
-    takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==True)).one()[0]
-    landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==False)).one()[0]
+    takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is True)).one()[0]
+    landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp >= today_beginning, TakeoffLanding.is_takeoff is False)).one()[0]
     sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date == date.today()).one()[0]
     sender_positions_total = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).one()[0]
 
     last_logbook_entries = db.session.query(Logbook).order_by(Logbook.reference_timestamp.desc()).limit(10)
-    return render_template("index.html",
+    return render_template(
+        "index.html",
         senders_today=senders_today,
         receivers_today=receivers_today,
         takeoffs_today=takeoffs_today,
@@ -80,6 +81,7 @@ def sender_detail():
 
     return render_template("sender_detail.html", title="Sender", sender=sender)
 
+
 @bp.route("/range_view.png")
 def range_view():
     import io
@@ -210,24 +212,28 @@ def download_flight():
 
     return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain")
 
+
 @bp.route("/sender_ranking.html")
 def sender_ranking():
     sender_statistics = db.session.query(SenderStatistic) \
-        .filter(db.and_(SenderStatistic.date==date.today(), SenderStatistic.is_trustworthy==True)) \
+        .filter(db.and_(SenderStatistic.date == date.today(), SenderStatistic.is_trustworthy is True)) \
         .order_by(SenderStatistic.max_distance.desc()) \
         .all()
 
-    return render_template("sender_ranking.html",
+    return render_template(
+        "sender_ranking.html",
         title="Sender Ranking",
         ranking=sender_statistics)
 
+
 @bp.route("/receiver_ranking.html")
 def receiver_ranking():
     receiver_statistics = db.session.query(ReceiverStatistic) \
-        .filter(db.and_(ReceiverStatistic.date==date.today(), ReceiverStatistic.is_trustworthy==True)) \
+        .filter(db.and_(ReceiverStatistic.date == date.today(), ReceiverStatistic.is_trustworthy is True)) \
         .order_by(ReceiverStatistic.max_distance.desc()) \
         .all()
 
-    return render_template("receiver_ranking.html",
+    return render_template(
+        "receiver_ranking.html",
         title="Receiver Ranking",
         ranking=receiver_statistics)

app/model/receiver.py

@@ -54,4 +54,3 @@ class Receiver(db.Model):
         )
         airports = [(airport, distance, azimuth) for airport, distance, azimuth in query]
         return airports
-

app/model/sender_direction_statistic.py

@@ -2,6 +2,7 @@ from app import db
 
 from sqlalchemy.dialects.postgresql import JSON
 
+
 class SenderDirectionStatistic(db.Model):
     __tablename__ = "sender_direction_statistics"
 

app/model/sender_info.py

@@ -4,6 +4,7 @@ from .aircraft_type import AircraftType
 
 #from sqlalchemy.dialects.postgresql import ENUM
 
+
 class SenderInfo(db.Model):
     __tablename__ = "sender_infos"
 

app/model/sender_position.py

@@ -60,4 +60,3 @@ class SenderPosition(db.Model):
     location_mgrs = db.Column(db.String(15))                # full mgrs (15 chars)
     location_mgrs_short = db.Column(db.String(9))           # reduced mgrs (9 chars), e.g. used for melissas range tool
     agl = db.Column(db.Float(precision=2))
-

app/model/sender_position_statistic.py

@@ -4,6 +4,7 @@ from .aircraft_type import AircraftType
 
 from sqlalchemy.dialects.postgresql import ENUM
 
+
 class SenderPositionStatistic(db.Model):
     __tablename__ = "sender_position_statistics"
 

app/tasks/orm_tasks.py

@@ -9,6 +9,7 @@ from app.collect.gateway import transfer_from_redis_to_database
 
 from app import db, celery
 
+
 @celery.task(name="transfer_to_database")
 def transfer_to_database():
     """Transfer APRS data from Redis to database."""

app/utils.py

@@ -137,6 +137,7 @@ def open_file(filename):
         f = open(filename, "rt", encoding="latin-1")
         return f
 
+
 def get_sql_trustworthy(source_table_alias):
     MIN_DISTANCE = 1000
     MAX_DISTANCE = 640000

config.py

@@ -1,5 +1,6 @@
 import os
 
+
 class BaseConfig:
     SECRET_KEY = "i-like-ogn"
 
@@ -16,6 +17,7 @@ class BaseConfig:
 
     APRS_USER = "OGNPYTHON"
 
+
 class DefaultConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "postgresql://postgres:postgres@localhost:5432/ogn")
     SQLALCHEMY_TRACK_MODIFICATIONS = False
@@ -37,11 +39,13 @@ class DefaultConfig(BaseConfig):
         #"purge_old_data": {"task": "purge_old_data", "schedule": timedelta(hours=1), "kwargs": {"max_hours": 48}},
     }
 
+
 class DevelopmentConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5432/ogn_test"
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SQLALCHEMY_ECHO = False
 
+
 configs = {
     'default': DefaultConfig,
     'development': DevelopmentConfig,

setup.cfg

@@ -1,2 +1,2 @@
 [flake8]
-ignore = F401, F841, E402, E501, E126
+ignore = F401, F841, E402, E501, E126, E265

tests/collect/test_logbook.py

@@ -76,7 +76,6 @@ class TestLogbook(TestBaseDB):
         self.assertEqual(entries[0].takeoff_airport_id, self.koenigsdorf.id)
         self.assertEqual(entries[0].landing_airport_id, self.koenigsdorf.id)
 
-
     @unittest.skip('needs information about airport timezone')
     def test_takeoff_and_landing_on_different_days(self):
         db.session.add(self.takeoff_koenigsdorf_dd0815)