Sender infos with country (#162)

* Added FK country_id to sender_infos

* Added flydenity to retrieve the country for registration

* Added country flag to sender_ranking, removed max_normalized_quality from rankings

* Bugfix: sort numbers first, strings always bottom

* Fixed testcases and refactored ddb import

* Small frontend improvements

README.md

@@ -146,11 +146,10 @@ Commands:
   export    Export data in several file formats.
   flights   Create 2D flight paths from data.
   gateway   Connection to APRS servers.
-  logbook   Handling of logbook data.
+  logbook   Handling of takeoff/landings and logbook data.
   routes    Show the routes for the app.
-  run       Runs a development server.
+  run       Run a development server.
-  shell     Runs a shell in the app context.
+  shell     Run a shell in the app context.
-  stats     Handling of statistical data.
 ```
 
 Most commands are command groups, so if you execute this command you will get further (sub)commands.

app/collect/database.py

@@ -1,9 +1,17 @@
+from io import StringIO
+import csv
+import requests
+
 from sqlalchemy.dialects.postgresql import insert
 from flask import current_app
 
+from flydenity import parser as flydenity_parser
+
 from app import db
-from app.model import SenderInfo, SenderInfoOrigin, Receiver
+from app.model import AircraftType, Country, Sender, SenderInfo, SenderInfoOrigin, Receiver
-from app.utils import get_ddb, get_flarmnet
+
+DDB_URL = "http://ddb.glidernet.org/download/?t=1"
+FLARMNET_URL = "http://www.flarmnet.org/files/data.fln"
 
 
 def upsert(model, rows, update_cols):
@@ -21,33 +29,91 @@ def upsert(model, rows, update_cols):
     return on_conflict_stmt
 
 
-def update_device_infos(address_origin, path=None):
+def read_ddb(csv_file=None):
-    if address_origin == SenderInfoOrigin.FLARMNET:
+    """Get SenderInfos. You can provide a local file path for user defined SenderInfos. Otherwise the SenderInfos will be fetched from official DDB."""
-        device_infos = get_flarmnet(fln_file=path)
+
+    if csv_file is None:
+        sender_info_origin = SenderInfoOrigin.OGN_DDB
+        r = requests.get(DDB_URL)
+        rows = "\n".join(i for i in r.text.splitlines() if i[0] != "#")
     else:
-        device_infos = get_ddb(csv_file=path)
+        sender_info_origin = SenderInfoOrigin.USER_DEFINED
+        r = open(csv_file, "r")
+        rows = "".join(i for i in r.readlines() if i[0] != "#")
+
+    data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
+
+    sender_info_dicts = []
+    for row in data:
+        sender_info_dicts.append({
+            'address_type': row[0],
+            'address': row[1],
+            'aircraft': row[2],
+            'registration': row[3],
+            'competition': row[4],
+            'tracked': row[5] == "Y",
+            'identified': row[6] == "Y",
+            'aircraft_type': AircraftType(int(row[7])),
+            'address_origin': sender_info_origin
+        })
+
+    return sender_info_dicts
+
+
+def read_flarmnet(fln_file=None):
+    if fln_file is None:
+        sender_info_origin = SenderInfoOrigin.FLARMNET
+        r = requests.get(FLARMNET_URL)
+        rows = [bytes.fromhex(line).decode("latin1") for line in r.text.split("\n") if len(line) == 173]
+    else:
+        sender_info_origin = SenderInfoOrigin.USER_DEFINED  # TODO: USER_DEFINED_FLARM ?
+        with open(fln_file, "r") as file:
+            rows = [bytes.fromhex(line.strip()).decode("latin1") for line in file.readlines() if len(line) == 173]
+
+    sender_info_dicts = []
+    for row in rows:
+        sender_info_dicts.append({
+            'address': row[0:6].strip(),
+            'aircraft': row[48:69].strip(),
+            'registration': row[69:76].strip(),
+            'competition': row[76:79].strip(),
+            'address_origin': sender_info_origin
+        })
+
+    return sender_info_dicts
+
+
+def merge_sender_infos(sender_info_dicts):
+    for sender_info_dict in sender_info_dicts:
+        statement = insert(SenderInfo) \
+            .values(**sender_info_dict) \
+            .on_conflict_do_update(
+                index_elements=['address', 'address_origin'],
+                set_=sender_info_dict)
+
+        db.session.execute(statement)
 
-    db.session.query(SenderInfo).filter(SenderInfo.address_origin == address_origin).delete(synchronize_session="fetch")
     db.session.commit()
 
-    for device_info in device_infos:
+    # update sender_infos FK countries
-        device_info.address_origin = address_origin
+    countries = {country.iso2: country for country in db.session.query(Country)}
 
-    db.session.bulk_save_objects(device_infos)
+    parser = flydenity_parser.ARParser()
+    for sender_info in db.session.query(SenderInfo).filter(SenderInfo.country_id == db.null()):
+        datasets = parser.parse(sender_info.registration, strict=True)
+        if datasets is None:
+            continue
+
+        for dataset in datasets:
+            if 'iso2' in dataset:
+                sender_info.country = countries[dataset['iso2']]
     db.session.commit()
 
-    return len(device_infos)
+    # Update sender_infos FK -> senders
+    upd = db.update(SenderInfo) \
+        .where(SenderInfo.address == Sender.address) \
+        .values(sender_id=Sender.id)
+    result = db.session.execute(upd)
+    db.session.commit()
 
-
+    return len(sender_info_dicts)
-def import_ddb(logger=None):
-    """Import registered devices from the DDB."""
-
-    if logger is None:
-        logger = current_app.logger
-
-    logger.info("Import registered devices fom the DDB...")
-    counter = update_device_infos(SenderInfoOrigin.OGN_DDB)
-
-    finish_message = "SenderInfo: {} inserted.".format(counter)
-    logger.info(finish_message)
-    return finish_message

app/commands/database.py

@@ -5,10 +5,10 @@ import click
 from datetime import datetime
 from sqlalchemy.sql import func
 
-from app.collect.database import update_device_infos
+from app.model import SenderPosition
-from app.model import SenderPosition, SenderInfoOrigin
 from app.utils import get_airports, get_days
 from app.collect.timescaledb_views import create_timescaledb_views, create_views
+from app.collect.database import read_ddb, read_flarmnet, merge_sender_infos
 
 from app import db
 
@@ -77,21 +77,17 @@ def drop(sure):
 
 
 @user_cli.command("import_ddb")
-def import_ddb():
+@click.option('--path', default=None, help='path to a local ddb file.')
+def import_ddb(path):
     """Import registered devices from the DDB."""
 
-    print("Import registered devices fom the DDB...")
+    if path is None:
-    counter = update_device_infos(SenderInfoOrigin.OGN_DDB)
+        print("Import registered devices fom the DDB...")
-    print("Imported %i devices." % counter)
+        sender_info_dicts = read_ddb()
-
+    else:
-
+        print("Import registered devices from '{}'...".format(path))
-@user_cli.command("import_file")
+        sender_info_dicts = read_ddb(csv_file=path)
-@click.argument("path")
+    counter = merge_sender_infos(sender_info_dicts)
-def import_file(path="tests/custom_ddb.txt"):
-    """Import registered devices from a local file."""
-
-    print("Import registered devices from '{}'...".format(path))
-    counter = update_device_infos(SenderInfoOrigin.USER_DEFINED, path=path)
     print("Imported %i devices." % counter)
 
 
@@ -101,7 +97,8 @@ def import_flarmnet(path=None):
     """Import registered devices from a local file."""
 
     print("Import registered devices from '{}'...".format("internet" if path is None else path))
-    counter = update_device_infos(SenderInfoOrigin.FLARMNET, path=path)
+    sender_info_dicts = read_flarmnet(path=path)
+    counter = merge_sender_infos(sender_info_dicts)
     print("Imported %i devices." % counter)
 
 

app/main/jinja_filters.py

@@ -12,13 +12,25 @@ def to_html_flag(obj):
         return ""
 
     if isinstance(obj, str):
-        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.lower()}" alt="{obj}"/>"""
+        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.lower()}" alt="{obj}"/> """
 
     elif isinstance(obj, Airport):
-        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.country_code.lower()}" alt="{obj.country_code}"/>"""
+        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.country_code.lower()}" alt="{obj.country_code}"/> """
 
     elif isinstance(obj, Country):
-        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.iso2.lower()}" alt="{obj.iso2}"/>"""
+        return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.iso2.lower()}" alt="{obj.iso2}"/> """
+
+    elif isinstance(obj, Sender):
+        if obj is not None and len(obj.infos) > 0 and obj.infos[0].country is not None:
+            return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.infos[0].country.iso2.lower()}" alt="{obj.infos[0].country.iso2}"/> """
+        else:
+            return ""
+
+    elif isinstance(obj, Receiver):
+        if obj.country:
+            return f"""<img src="{url_for('static', filename='img/Transparent.gif')}" class="flag flag-{obj.country.iso2.lower()}" alt="{obj.country.iso2}"/> """
+        else:
+            return ""
 
 
 @bp.app_template_filter()

app/model/sender_info.py

@@ -10,7 +10,7 @@ class SenderInfo(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     address = db.Column(db.String(6), index=True)
-    address_type = None
+    address_type = db.Column(db.String)
     aircraft = db.Column(db.String)
     registration = db.Column(db.String(7))
     competition = db.Column(db.String(3))
@@ -24,6 +24,11 @@ class SenderInfo(db.Model):
     sender_id = db.Column(db.Integer, db.ForeignKey("senders.id"), index=True)
     sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("infos", order_by=address_origin))
 
+    country_id = db.Column(db.Integer, db.ForeignKey("countries.gid"), index=True)
+    country = db.relationship("Country", foreign_keys=[country_id], backref=db.backref("sender_infos", order_by=address_origin))
+
+    __table_args__ = (db.Index('idx_sender_infos_address_address_origin_uc', 'address', 'address_origin', unique=True), )
+
     def __repr__(self):
         return "<SenderInfo: %s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
             self.address_type,

app/tasks/orm_tasks.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from app.collect.logbook import update_takeoff_landings as logbook_update_takeoff_landings, update_logbook as logbook_update
 from app.collect.logbook import update_max_altitudes as logbook_update_max_altitudes
 
-from app.collect.database import import_ddb as device_infos_import_ddb
+from app.collect.database import read_ddb, merge_sender_infos
 
 from app.collect.gateway import transfer_from_redis_to_database
 
@@ -48,5 +48,6 @@ def update_logbook_max_altitude():
 def import_ddb():
     """Import registered devices from the DDB."""
 
-    result = device_infos_import_ddb()
+    sender_info_dicts = read_ddb()
+    result = merge_sender_infos(sender_info_dicts)
     return result

app/templates/airport_detail.html

@@ -2,14 +2,12 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
 <div class="container">
 
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Airport Details</h3></div>
 <table class="datatable table table-striped table-bordered">
-	<tr><td>Name:</td><td>{{ airport|to_html_flag|safe }} {{ airport.name }}</td></tr>
+	<tr><td>Name:</td><td>{{ airport|to_html_flag|safe }}{{ airport.name }}</td></tr>
 	<tr><td>Code:</td><td>{{ airport.code }}</td></tr>
 	<tr><td>Altitude:</td><td>{{ airport.altitude|int }} m</td></tr>
 	<tr><td>Style:</td><td>{{ airport.style }}</td></tr>

app/templates/airports.html

@@ -2,8 +2,6 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
 <div class="container">
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Airports</h3></div>

app/templates/base.html

@@ -4,6 +4,7 @@
     {{ super() }}
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.31.3/css/theme.default.min.css">
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.31.3/css/theme.bootstrap_3.min.css">
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
 {% endblock %}
 
 {% block title %}

app/templates/index.html

@@ -1,6 +1,7 @@
 {% extends "base.html" %}
 
 {% block content %}
+
 <div class="container">
 
 
@@ -63,7 +64,7 @@
     <tr>
       <td>{{ loop.index }}</td>
       <td>{% if ns.mydate != entry.reference_timestamp.strftime('%Y-%m-%d') %}{% set ns.mydate = entry.reference_timestamp.strftime('%Y-%m-%d') %}{{ ns.mydate }}{% endif %}</td>
-      <td>{{ entry.sender|to_html_link|safe }}</td>
+      <td>{{ entry.sender|to_html_flag|safe }}{{ entry.sender|to_html_link|safe }}</td>
       <td>{% if entry.sender.infos|length > 0 and entry.sender.infos[0].aircraft|length %}{{ entry.sender.infos[0].aircraft }}{% else %}-{% endif %}</td>
       <td>{% if entry.takeoff_airport is not none %}<a href="{{ url_for('main.logbooks', country=entry.takeoff_airport.country_code, airport_id=entry.takeoff_airport.id, date=entry.reference_timestamp.strftime('%Y-%m-%d')) }}">{{ entry.takeoff_airport.name }}</a>{% endif %}</td>
       <td>{% if entry.landing_airport is not none %}<a href="{{ url_for('main.logbooks', country=entry.landing_airport.country_code, airport_id=entry.landing_airport.id, date=entry.reference_timestamp.strftime('%Y-%m-%d')) }}">{{ entry.landing_airport.name }}</a>{% endif %}</td>

app/templates/logbooks.html

@@ -2,8 +2,6 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
 <div class="container">
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Logbook</h3></div>
@@ -61,8 +59,8 @@
         <td>{% if entry.duration is not none %}{{ entry.duration }}{% endif %}</td>
         <td>{% if entry.max_altitude is not none %}{{ '%d' | format(entry.max_altitude - entry.takeoff_airport.altitude) }} m{% endif %}</td>
         <td>
-        	{% if entry.takeoff_airport is not none and entry.takeoff_airport.id != sel_airport_id %}Take Off: {{ entry.takeoff_airport|to_html_flag|safe }} <a href="{{ url_for('main.logbooks', country=entry.takeoff_airport.country_code, airport_id=entry.takeoff_airport.id, date=sel_date) }}">{{ entry.takeoff_airport.name }}</a>
+        	{% if entry.takeoff_airport is not none and entry.takeoff_airport.id != sel_airport_id %}Take Off: {{ entry.takeoff_airport|to_html_flag|safe }}<a href="{{ url_for('main.logbooks', country=entry.takeoff_airport.country_code, airport_id=entry.takeoff_airport.id, date=sel_date) }}">{{ entry.takeoff_airport.name }}</a>
-	        {% elif entry.landing_airport is not none and entry.landing_airport.id != sel_airport_id %}Landing: {{ entry.landing_airport|to_html_flag|safe }} <a href="{{ url_for('main.logbooks', country=entry.takeoff_airport.country_code, airport_id=entry.landing_airport.id, date=sel_date) }}">{{ entry.landing_airport.name }}</a>
+	        {% elif entry.landing_airport is not none and entry.landing_airport.id != sel_airport_id %}Landing: {{ entry.landing_airport|to_html_flag|safe }}<a href="{{ url_for('main.logbooks', country=entry.takeoff_airport.country_code, airport_id=entry.landing_airport.id, date=sel_date) }}">{{ entry.landing_airport.name }}</a>
 	        {% endif %}
 	    </td>
 	</tr>

app/templates/receiver_detail.html

@@ -2,15 +2,12 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
-
 <div class="container">
 
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Receiver Details</h3></div>
 <table class="datatable table table-striped table-bordered">
-	<tr><td>Name:</td><td>{{ receiver.country|to_html_flag|safe }} {{ receiver.name }}</td></tr>
+	<tr><td>Name:</td><td>{{ receiver|to_html_flag|safe }}{{ receiver.name }}</td></tr>
 	<tr><td>Airport:</td>
 		<td>{% if receiver.airport is not none %}{{ receiver.airport|to_html_flag|safe }}
 			<a href="{{ url_for('main.airport_detail', airport_id=receiver.airport.id) }}">{{ receiver.airport.name }}</a>

app/templates/receiver_ranking.html

@@ -2,8 +2,6 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
 <div class="container">
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Receiver Ranking</h3></div>
@@ -28,7 +26,6 @@
         <th>Name</th>
         <th>Airport</th>
         <th class="text-right">Distance [km]</th>
-        <th class="text-right">Normalized signal quality [dB]</th>
         <th class="text-right">Senders</th>
         <th class="text-right">Coverages</th>
         <th class="text-right">Messages</th>
@@ -41,10 +38,9 @@
         <td class="text-right">{{ today }}</td>
         <td class="text-right">{% if yesterday is none %}(new){% elif yesterday - today > 0 %}<span class="text-success"><i class="fa fa-long-arrow-up"></i>{{ yesterday - today }}</span>{% elif yesterday - today < 0 %}<span class="text-danger"><i class="fa fa-long-arrow-down"></i>{{ today - yesterday }}</span>{% endif %}</td>
         <td class="text-right">{% if current is not none %}{{ current }}{% else %}-{% endif %}</td>
-        <td>{{ receiver.country|to_html_flag|safe }} {{ receiver|to_html_link|safe }}</td>
+        <td>{{ receiver|to_html_flag|safe }}{{ receiver|to_html_link|safe }}</td>
         <td>{{ receiver.airport|to_html_link|safe }}</td>
         <td class="text-right">{% if ranking is not none %}{{ '%0.1f' | format(ranking.max_distance/1000.0) }}{% else %}-{% endif %}</td>
-        <td class="text-right">{% if ranking is not none %}{{ '%0.1f' | format(ranking.max_normalized_quality) }}{% else %}-{% endif %}</td>
         <td class="text-right">{% if ranking is not none %}{{ ranking.senders_count }}{% else %}-{% endif %}</td>
         <td class="text-right">{% if ranking is not none %}{{ ranking.coverages_count }}{% else %}-{% endif %}</td>
         <td class="text-right">{% if ranking is not none %}{{ ranking.messages_count }}{% else %}-{% endif %}</td>
@@ -61,7 +57,12 @@
   {{ super() }}
   <script>
     $(function() {
-      $("#myTable").tablesorter({sortList: [[0,0]], theme:"bootstrap", headerTemplate:"{content} {icon}", widgets:["uitheme"]});
+      $("#myTable").tablesorter({
+          stringTo:"bottom",
+          theme:"bootstrap",
+          headerTemplate:"{content} {icon}",
+          widgets:["uitheme"]
+        });
     });
   </script>
 {% endblock %}

app/templates/receivers.html

@@ -2,8 +2,6 @@
 
 {% block content %}
 
-<link rel="stylesheet" href="{{ url_for('static', filename='css/flags/flags.css') }}"/>
-
 <div class="container">
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Receivers</h3></div>
@@ -38,7 +36,7 @@
       {% for receiver in receivers %}
       <tr>
         <td>{{ loop.index }}</td>
-        <td>{{ receiver.country|to_html_flag|safe }}</td>
+        <td>{{ receiver|to_html_flag|safe }}</td>
         <td>{{ receiver|to_html_link|safe }}</td>
         <td>{{ receiver.airport|to_html_link|safe }}</td>
         <td>{{ receiver.altitude|int }} m</td>

app/templates/sender_detail.html

@@ -32,7 +32,7 @@
 	{% for info in sender.infos %}
 	<tr>
 		<td>{{ info.aircraft }}</td>
-		<td>{{ info.registration }}</td>
+		<td>{{ info|to_html_flag|safe }}{{ info.registration }}</td>
 		<td>{{ info.competition }}</td>
 		<td>{{ info.aircraft_type.name }}</td>
 		<td>{{ info.address_origin.name }}</td>

app/templates/sender_ranking.html

@@ -1,6 +1,7 @@
 {% extends "base.html" %}
 
 {% block content %}
+
 <div class="container">
 <div class="panel panel-success">
 <div class="panel-heading"><h3 class="panel-title">Sender Ranking</h3></div>
@@ -12,7 +13,6 @@
         <th>Name</th>
         <th>Aircraft</th>
         <th class="text-right">Maximum distance [km]</th>
-        <th class="text-right">Maximal normalized signal quality [dB]</th>
         <th class="text-right">Receiver counter</th>
         <th class="text-right">Coverage counter</th>
         <th class="text-right">Message counter</th>
@@ -23,10 +23,9 @@
       {% for entry in ranking %}
       <tr>
         <td>{{ loop.index }}</td>
-        <td>{{ entry.sender|to_html_link|safe }}</a></td>
+        <td>{{ entry.sender|to_html_flag|safe }}{{ entry.sender|to_html_link|safe }}</a></td>
         <td>{% if entry.sender.infos|length > 0 %}{{ entry.sender.infos[0].aircraft }}{% else %}-{% endif %}</td>
         <td class="text-right">{{ '%0.1f' | format(entry.max_distance/1000.0) }}</td>
-        <td class="text-right">{{ '%0.1f' | format(entry.max_normalized_quality) }}</td>
         <td class="text-right">{{ entry.receivers_count }}</td>
         <td class="text-right">{{ entry.coverages_count }}</td>
         <td class="text-right">{{ entry.messages_count }}</td>
@@ -44,7 +43,12 @@
   {{ super() }}
   <script>
     $(function() {
-      $("#myTable").tablesorter({sortList: [[0,0]], theme:"bootstrap", headerTemplate:"{content} {icon}", widgets:["uitheme"]});
+      $("#myTable").tablesorter({
+          stringTo:"bottom",
+          theme:"bootstrap",
+          headerTemplate:"{content} {icon}",
+          widgets:["uitheme"]
+        });
     });
   </script>
 {% endblock %}

app/utils.py

@@ -1,21 +1,14 @@
-import csv
 import gzip
-from io import StringIO
 from datetime import datetime, timedelta
 
 from flask import current_app
 
 from aerofiles.seeyou import Reader
 from ogn.parser.utils import FEETS_TO_METER
-import requests
 
 from .model import AircraftType, SenderInfoOrigin, SenderInfo, Airport, Location
 
 
-DDB_URL = "http://ddb.glidernet.org/download/?t=1"
-FLARMNET_URL = "http://www.flarmnet.org/files/data.fln"
-
-
 address_prefixes = {"F": "FLR", "O": "OGN", "I": "ICA"}
 
 nm2m = 1852
@@ -33,60 +26,11 @@ def date_to_timestamps(date):
     return (start, end)
 
 
-def get_ddb(csv_file=None, address_origin=SenderInfoOrigin.UNKNOWN):
+def get_trackable(sender_info_dicts):
-    if csv_file is None:
-        r = requests.get(DDB_URL)
-        rows = "\n".join(i for i in r.text.splitlines() if i[0] != "#")
-    else:
-        r = open(csv_file, "r")
-        rows = "".join(i for i in r.readlines() if i[0] != "#")
-
-    data = csv.reader(StringIO(rows), quotechar="'", quoting=csv.QUOTE_ALL)
-
-    sender_infos = list()
-    for row in data:
-        sender_info = SenderInfo()
-        sender_info.address_type = row[0]
-        sender_info.address = row[1]
-        sender_info.aircraft = row[2]
-        sender_info.registration = row[3]
-        sender_info.competition = row[4]
-        sender_info.tracked = row[5] == "Y"
-        sender_info.identified = row[6] == "Y"
-        sender_info.aircraft_type = AircraftType(int(row[7]))
-        sender_info.address_origin = address_origin
-
-        sender_infos.append(sender_info)
-
-    return sender_infos
-
-
-def get_flarmnet(fln_file=None, address_origin=SenderInfoOrigin.FLARMNET):
-    if fln_file is None:
-        r = requests.get(FLARMNET_URL)
-        rows = [bytes.fromhex(line).decode("latin1") for line in r.text.split("\n") if len(line) == 173]
-    else:
-        with open(fln_file, "r") as file:
-            rows = [bytes.fromhex(line.strip()).decode("latin1") for line in file.readlines() if len(line) == 173]
-
-    sender_infos = list()
-    for row in rows:
-        sender_info = SenderInfo()
-        sender_info.address = row[0:6].strip()
-        sender_info.aircraft = row[48:69].strip()
-        sender_info.registration = row[69:76].strip()
-        sender_info.competition = row[76:79].strip()
-
-        sender_infos.append(sender_info)
-
-    return sender_infos
-
-
-def get_trackable(ddb):
     result = []
-    for i in ddb:
+    for sender_info_dict in sender_info_dicts:
-        if i.tracked and i.address_type in address_prefixes:
+        if sender_info_dict['tracked'] and sender_info_dict['address_type'] in address_prefixes:
-            result.append("{}{}".format(address_prefixes[i.address_type], i.address))
+            result.append("{}{}".format(address_prefixes[sender_info_dict['address_type']], sender_info_dict['address']))
     return result
 
 

migrations/versions/2ab0bbb8b49d_added_constraint_to_senderinfo.py

@@ -0,0 +1,30 @@
+"""Added Constraint to SenderInfo
+
+Revision ID: 2ab0bbb8b49d
+Revises: a72b2205b55c
+Create Date: 2020-12-11 23:27:16.497547
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2ab0bbb8b49d'
+down_revision = 'a72b2205b55c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('sender_infos', sa.Column('address_type', sa.VARCHAR(), nullable=True))
+    op.create_index('idx_sender_infos_address_address_origin_uc', 'sender_infos', ['address', 'address_origin'], unique=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('idx_sender_infos_address_address_origin_uc', table_name='sender_infos')
+    op.drop_column('sender_infos', 'address_type')
+    # ### end Alembic commands ###

migrations/versions/a72b2205b55c_added_country_relation_to_senderinfo.py

@@ -0,0 +1,32 @@
+"""Added country relation to SenderInfo
+
+Revision ID: a72b2205b55c
+Revises: f3afd6197391
+Create Date: 2020-12-08 18:03:10.131819
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a72b2205b55c'
+down_revision = 'f3afd6197391'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('sender_infos', sa.Column('country_id', sa.Integer(), nullable=True))
+    op.create_index(op.f('ix_sender_infos_country_id'), 'sender_infos', ['country_id'], unique=False)
+    op.create_foreign_key(None, 'sender_infos', 'countries', ['country_id'], ['gid'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'sender_infos', type_='foreignkey')
+    op.drop_index(op.f('ix_sender_infos_country_id'), table_name='sender_infos')
+    op.drop_column('sender_infos', 'country_id')
+    # ### end Alembic commands ###

setup.py

@@ -58,7 +58,8 @@ setup(
         'requests==2.25.0',
         'matplotlib==3.3.3',
         'bokeh==2.2.3',
-        'pandas==1.1.5'
+        'pandas==1.1.5',
+        'flydenity==0.1.5'
     ],
     test_require=[
         'pytest==5.0.1',

tests/base.py

@@ -33,6 +33,11 @@ class TestBaseDB(unittest.TestCase):
         db.session.execute("SELECT create_hypertable('receiver_statuses', 'reference_timestamp', chunk_time_interval => interval '1 day', if_not_exists => TRUE);")
         db.session.commit()
 
+        # ... and insert some countries
+        db.session.execute("INSERT INTO countries(name, iso2) VALUES ('Germany', 'DE');")
+        db.session.execute("INSERT INTO countries(name, iso2) VALUES ('Austria', 'AT');")
+        db.session.execute("INSERT INTO countries(name, iso2) VALUES ('United Kingdom', 'GB');")
+
     def tearDown(self):
         db.session.remove()
 

tests/commands/test_database.py

@@ -3,15 +3,16 @@ import os
 
 from flask import current_app
 from app.model import SenderInfo
-from app.commands.database import import_file
+from app.commands.database import import_ddb
 
 from tests.base import TestBaseDB, db
 
 
 class TestDatabase(TestBaseDB):
-    def test_import_ddb_file(self):
+    def test_import_ddb(self):
         runner = current_app.test_cli_runner()
-        result = runner.invoke(import_file, [os.path.dirname(__file__) + "/../custom_ddb.txt"])
+        ddb_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../custom_ddb.txt"))
+        result = runner.invoke(import_ddb, ['--path', ddb_path])
         self.assertEqual(result.exit_code, 0)
 
         sender_infos = db.session.query(SenderInfo).all()

tests/test_utils.py

@@ -3,7 +3,8 @@ import unittest
 from datetime import date
 
 from app.model import AircraftType
-from app.utils import get_days, get_ddb, get_trackable, get_airports
+from app.utils import get_days, get_trackable, get_airports
+from app.commands.database import read_ddb
 
 
 class TestStringMethods(unittest.TestCase):
@@ -14,25 +15,25 @@ class TestStringMethods(unittest.TestCase):
         self.assertEqual(days, [date(2018, 2, 27), date(2018, 2, 28), date(2018, 3, 1), date(2018, 3, 2)])
 
     def test_get_devices(self):
-        devices = get_ddb()
+        sender_infos = read_ddb()
-        self.assertGreater(len(devices), 1000)
+        self.assertGreater(len(sender_infos), 1000)
 
     def test_get_ddb_from_file(self):
-        devices = get_ddb(os.path.dirname(__file__) + "/custom_ddb.txt")
+        sender_infos = read_ddb(os.path.dirname(__file__) + "/custom_ddb.txt")
-        self.assertEqual(len(devices), 6)
+        self.assertEqual(len(sender_infos), 6)
-        device = devices[0]
+        sender_info = sender_infos[0]
 
-        self.assertEqual(device.address, "DD4711")
+        self.assertEqual(sender_info['address'], "DD4711")
-        self.assertEqual(device.aircraft, "HK36 TTC")
+        self.assertEqual(sender_info['aircraft'], "HK36 TTC")
-        self.assertEqual(device.registration, "D-EULE")
+        self.assertEqual(sender_info['registration'], "D-EULE")
-        self.assertEqual(device.competition, "CU")
+        self.assertEqual(sender_info['competition'], "CU")
-        self.assertTrue(device.tracked)
+        self.assertTrue(sender_info['tracked'])
-        self.assertTrue(device.identified)
+        self.assertTrue(sender_info['identified'])
-        self.assertEqual(device.aircraft_type, AircraftType.GLIDER_OR_MOTOR_GLIDER)
+        self.assertEqual(sender_info['aircraft_type'], AircraftType.GLIDER_OR_MOTOR_GLIDER)
 
     def test_get_trackable(self):
-        devices = get_ddb(os.path.dirname(__file__) + "/custom_ddb.txt")
+        sender_infos = read_ddb(os.path.dirname(__file__) + "/custom_ddb.txt")
-        trackable = get_trackable(devices)
+        trackable = get_trackable(sender_infos)
         self.assertEqual(len(trackable), 4)
         self.assertIn("FLRDD4711", trackable)
         self.assertIn("FLRDD0815", trackable)