Added Flask-Redis

Repaired test cases

app/__init__.py

@@ -4,33 +4,56 @@ from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
 from flask_caching import Cache
 from celery import Celery
+from flask_redis import FlaskRedis
+
+from config import configs
 
 bootstrap = Bootstrap()
 db = SQLAlchemy()
 migrate = Migrate()
 cache = Cache()
-celery = Celery(__name__, broker='redis://localhost:6379/0')
+redis_client = FlaskRedis()
+celery = Celery()
 
 
-def create_app(config_name='development'):
+def create_app(config_name='default'):
     # Initialize Flask
     app = Flask(__name__)
 
     # Load the configuration
-    if config_name == 'testing':
+    configuration = configs[config_name]
-        app.config.from_object('app.config.test')
+    app.config.from_object(configuration)
-    else:
-        app.config.from_object('app.config.default')
     app.config.from_envvar("OGN_CONFIG_MODULE", silent=True)
-    celery.config_from_object(app.config)
+    
-
     # Initialize other things
     bootstrap.init_app(app)
     db.init_app(app)
     migrate.init_app(app, db)
     cache.init_app(app)
-
+    redis_client.init_app(app)
+    
+    init_celery(app)
+    
     from app.main import bp as bp_main
     app.register_blueprint(bp_main)
 
     return app
+
+def init_celery(app):
+    celery.conf.broker_url = app.config['CELERY_BROKER_URL']
+    celery.conf.result_backend = app.config['CELERY_RESULT_BACKEND']
+    celery.conf.update(app.config)
+
+    class ContextTask(celery.Task):
+        """Make celery tasks work with Flask app context"""
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return self.run(*args, **kwargs)
+
+    celery.Task = ContextTask
+    return celery
+
+# Do we need this? Otherwise I cant the celery worker run...
+app = create_app()
+from app.gateway.bulkimport import DbFeeder
+from app.collect.celery_tasks import *

app/collect/celery_tasks.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+from flask import current_app
+
+from app import create_app
+from app import redis_client, celery
+
+from app.gateway.bulkimport import DbFeeder
+
+
+@celery.task(name="transfer_beacons_to_database")
+def transfer_beacons_to_database():
+    """Transfer beacons from redis to TimescaleDB."""
+
+    counter = 0
+    with DbFeeder() as feeder:
+        for key in redis_client.scan_iter(match="ogn-python *"):
+            value = redis_client.get(key)
+            if value is None:
+                redis_client.delete(key)
+                continue
+            
+            reference_timestamp = datetime.strptime(key[11:].decode('utf-8'), "%Y-%m-%d %H:%M:%S.%f")
+            aprs_string = value.decode('utf-8')
+            redis_client.delete(key)
+            
+            feeder.add(aprs_string, reference_timestamp=reference_timestamp)
+            counter += 1
+    
+    return f"Beacons transfered from redis to TimescaleDB: {counter}"
+
+if __name__ == '__main__':
+    app = create_app()
+    with app.app_context():
+        result = transfer_beacons_to_database.delay()
+        print(result)

app/commands/gateway.py

@@ -1,13 +1,15 @@
 import os
-import datetime
+from datetime import datetime, timezone
 
 from flask import current_app
 from flask.cli import AppGroup
 import click
+from tqdm import tqdm
 
 from ogn.client import AprsClient
 
-from app.gateway.bulkimport import convert, DbFeeder
+from app import redis_client
+from app.gateway.bulkimport import convert, calculate
 
 user_cli = AppGroup("gateway")
 user_cli.help = "Connection to APRS servers."
@@ -15,7 +17,7 @@ user_cli.help = "Connection to APRS servers."
 
 @user_cli.command("run")
 def run(aprs_user="anon-dev"):
-    """Run the aprs client and feed the DB with incoming data."""
+    """Run the aprs client and feed the redis db with incoming data."""
 
     # User input validation
     if len(aprs_user) < 3 or len(aprs_user) > 9:
@@ -26,24 +28,70 @@ def run(aprs_user="anon-dev"):
     client = AprsClient(aprs_user)
     client.connect()
 
-    with DbFeeder(prefix='continuous_import', reference_timestamp=datetime.utcnow, reference_timestamp_autoupdate=True) as feeder:
+    def insert_into_redis(aprs_string):
-        try:
+        redis_client.set(f"ogn-python {datetime.utcnow()}", aprs_string.strip(), ex=100)
-            client.run(callback=lambda x: feeder.add(x), autoreconnect=True)
+        insert_into_redis.beacon_counter += 1
-        except KeyboardInterrupt:
+        
-            current_app.logger.warning("\nStop ogn gateway")
+        delta = (datetime.utcnow() - insert_into_redis.last_update).total_seconds()
+        if delta >= 60.0:
+            print(f"{insert_into_redis.beacon_counter/delta:05.1f}/s")
+            insert_into_redis.last_update = datetime.utcnow()
+            insert_into_redis.beacon_counter = 0
+    
+    insert_into_redis.beacon_counter = 0
+    insert_into_redis.last_update = datetime.utcnow()
+
+    try:
+        client.run(callback=insert_into_redis, autoreconnect=True)
+    except KeyboardInterrupt:
+        current_app.logger.warning("\nStop ogn gateway")
 
     client.disconnect()
 
+@user_cli.command("printout")
+def printout():
+    """Run the aprs client and just print out the data stream."""
+    
+    current_app.logger.warning("Start ogn gateway")
+    client = AprsClient("anon-dev")
+    client.connect()
+
+    try:
+        client.run(callback=lambda x: print(f"{datetime.utcnow()}: {x}"), autoreconnect=True)
+    except KeyboardInterrupt:
+        current_app.logger.warning("\nStop ogn gateway")
+
+    client.disconnect()
 
 @user_cli.command("convert")
 @click.argument("path")
 def file_import(path):
     """Convert APRS logfiles into csv files for fast bulk import."""
 
-    logfiles = []
     for (root, dirs, files) in os.walk(path):
         for file in sorted(files):
-            logfiles.append(os.path.join(root, file))
+            print(file)
+            convert(os.path.join(root, file))
 
-    for logfile in logfiles:
+
-        convert(logfile)
+@user_cli.command("calculate")
+@click.argument("path")
+def file_calculate(path):
+    """Import csv files, calculate geographic features (distance, radial, agl, ...) and make data distinct."""
+
+    file_tuples = []
+    for (root, dirs, files) in os.walk(path):
+        for file in sorted(files):
+            if file.startswith('aircraft_beacons') and file.endswith('.csv.gz'):
+                ab_filename = os.path.join(root, file)
+                rb_filename = os.path.join(root, 'receiver' + file[8:])
+                target_filename = os.path.join(root, file + '2')
+                if os.path.isfile(target_filename):
+                    print("Outputfile {} already exists. Skipping".format(target_filename))
+                else:
+                    file_tuples.append((ab_filename, rb_filename, target_filename))
+
+    pbar = tqdm(file_tuples)
+    for file_tuple in pbar:
+        pbar.set_description("Converting {}".format(file_tuple[0]))
+        calculate(file_tuple[0], file_tuple[1], file_tuple[2])

app/gateway/bulkimport.py

@@ -60,7 +60,7 @@ AIRCRAFT_POSITION_BEACON_FIELDS = [
     "location_mgrs_short",
     "agl",
 
-    "reference_timestamp"
+    "reference_timestamp",
 ]
 
 RECEIVER_POSITION_BEACON_FIELDS = [
@@ -71,7 +71,7 @@ RECEIVER_POSITION_BEACON_FIELDS = [
     "receiver_name",
     "timestamp",
 
-    "reference_timestamp"
+    "reference_timestamp",
 ]
 
 RECEIVER_STATUS_BEACON_FIELDS = [
@@ -82,6 +82,8 @@ RECEIVER_STATUS_BEACON_FIELDS = [
 
     "version",
     "platform",
+    
+    "reference_timestamp",
 ]
 
 
@@ -103,24 +105,17 @@ def initial_file_scan(file):
 
 
 class StringConverter:
-    def __init__(self, reference_timestamp, auto_update_timestamp):
+    mgrs = MGRS()
-        self.reference_timestamp = reference_timestamp
+    
-        self.auto_update_timestamp = auto_update_timestamp
-
-        self.mgrs = MGRS()
-
     def __enter__(self):
         return self
 
     def __exit__(self, *args):
         pass
 
-    def _convert(self, raw_string):
+    def _convert(self, raw_string, reference_timestamp):
-        if raw_string.strip() == '':
-            return
-
         try:
-            message = parse(raw_string, reference_timestamp=self.reference_timestamp)
+            message = parse(raw_string, reference_timestamp)
         except NotImplementedError as e:
             current_app.logger.error("No parser implemented for message: {}".format(raw_string))
             return
@@ -136,11 +131,7 @@ class StringConverter:
             current_app.logger.error("Other Exception with string: {}".format(raw_string))
             return
 
-        if message['aprs_type'] not in ('server', 'position', 'status'):
+        if message['aprs_type'] not in ('position', 'status'):
-            return
-
-        elif message['aprs_type'] == 'server' and self.auto_update_timestamp is True:
-            self.reference_timestamp = message['timestamp']
             return
 
         elif message['aprs_type'] == 'position':
@@ -215,7 +206,7 @@ class StringConverter:
         return csv_string
 
     def _get_receiver_status_beacon_csv_string(self, message, none_character=''):
-        csv_string = "{0},{1},{2},{3},{4},{5}\n".format(
+        csv_string = "{0},{1},{2},{3},{4},{5},{6}\n".format(
             message['name'],
             message['dstcall'],
             message['receiver_name'],
@@ -223,6 +214,8 @@ class StringConverter:
 
             message['version'] if 'version' in message and message['version'] else none_character,
             message['platform'] if 'platform' in message and message['platform'] else none_character,
+            
+            message['reference_timestamp']
         )
         return csv_string
 
@@ -257,10 +250,7 @@ class FileFeeder(StringConverter):
 
 
 class DbFeeder(StringConverter):
-    def __init__(self, reference_timestamp, reference_timestamp_autoupdate):
+    def __init__(self):
-        self.reference_timestamp = reference_timestamp
-        self.reference_timestamp_autoupdate = reference_timestamp_autoupdate
-
         self.aircraft_position_beacons_buffer = StringIO()
         self.aircraft_status_beacons_buffer = StringIO()
         self.receiver_position_beacons_buffer = StringIO()
@@ -268,18 +258,17 @@ class DbFeeder(StringConverter):
 
         self.last_flush = datetime.utcnow()
 
-        super().__init__(reference_timestamp, reference_timestamp_autoupdate)
-
     def __exit__(self, *args):
         self.flush()
 
-    def add(self, raw_string):
+    def add(self, raw_string, reference_timestamp):
         raw_string = raw_string.strip()
 
-        message = self._convert(raw_string)
+        message = self._convert(raw_string, reference_timestamp=reference_timestamp)
         if not message:
             return
 
+        message['reference_timestamp'] = reference_timestamp
         if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and message['aprs_type'] == 'position':
             csv_string = self._get_aircraft_position_beacon_csv_string(message, none_character=r'\N')
             self.aircraft_position_beacons_buffer.write(csv_string)
@@ -294,10 +283,6 @@ class DbFeeder(StringConverter):
         else:
             current_app.logger.error(f"Not supported. beacon_type: '{message['beacon_type']}', aprs_type: '{message['aprs_type']}', skipped: {raw_string}")
 
-        if datetime.utcnow() - self.last_flush >= timedelta(seconds=1):
-            self.flush()
-            self.last_flush = datetime.utcnow()
-
     def _flush_position_beacons(self):
         connection = db.engine.raw_connection()
         cursor = connection.cursor()
@@ -447,14 +432,15 @@ class DbFeeder(StringConverter):
 
         # Update receiver_beacons
         cursor.execute("""
-            INSERT INTO receiver_beacons AS rb (name, dstcall, receiver_name, timestamp, version, platform)
+            INSERT INTO receiver_beacons AS rb (name, dstcall, receiver_name, timestamp, version, platform, reference_timestamp)
             SELECT DISTINCT ON (rsbt.name)
                 rsbt.name,
                 rsbt.dstcall,
                 rsbt.receiver_name,
                 rsbt.timestamp,
                 rsbt.version,
-                rsbt.platform
+                rsbt.platform,
+                rsbt.reference_timestamp
             FROM receiver_status_beacons_temp AS rsbt,
             (
                 SELECT

setup.py

@@ -43,6 +43,7 @@ setup(
         'Flask-Caching==1.9.0',
         'geopy==2.0.0',
         'celery==5.0.2',
+        'Flask-Redis==0.4.0',
         'redis==3.5.3',
         'aerofiles==1.0.0',
         'geoalchemy2==0.8.4',

tests/base.py

@@ -18,6 +18,8 @@ class TestBaseDB(unittest.TestCase):
 
         # ... and create them again
         db.session.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
+        db.session.commit()
+
         db.create_all()
         db.session.commit()
 
@@ -28,6 +30,7 @@ class TestBaseDB(unittest.TestCase):
         db.session.execute("CREATE EXTENSION IF NOT EXISTS timescaledb;")
         db.session.execute("SELECT create_hypertable('aircraft_beacons', 'timestamp', chunk_time_interval => interval '1 day', if_not_exists => TRUE);")
         db.session.execute("SELECT create_hypertable('receiver_beacons', 'timestamp', chunk_time_interval => interval '1 day', if_not_exists => TRUE);")
+        db.session.commit()
 
     def tearDown(self):
         db.session.remove()

tests/gateway/beacon_data/valid_beacon_data/aprs_aircraft.txt

@@ -0,0 +1,15 @@
+# Until OGN software 0.2.6 all beacons (flarms, ogn trackers and receivers) have the dstcall "APRS"
+# These are example beacons for flarms and ogn trackers
+#
+FLRDDA5BA>APRS,qAS,LFMX:/165829h4415.41N/00600.03E'342/049/A=005524 id0ADDA5BA -454fpm -1.1rot 8.8dB 0e +51.2kHz gps4x5
+ICA4B0E3A>APRS,qAS,Letzi:/165319h4711.75N\00802.59E^327/149/A=006498 id154B0E3A -3959fpm +0.5rot 9.0dB 0e -6.3kHz gps1x3
+FLRDDB091>APRS,qAS,Letzi:/165831h4740.04N/00806.01EX152/124/A=004881 id06DD8E80 +198fpm +0.0rot 6.5dB 13e +4.0kHz gps3x4
+FLRDDDD33>APRS,qAS,LFNF:/165341h4344.27N/00547.41E'/A=000886 id06DDDD33 +020fpm +0.0rot 20.8dB 0e -14.3kHz gps3x4
+FLRDDE026>APRS,qAS,LFNF:/165341h4358.58N/00553.89E'204/055/A=005048 id06DDE026 +257fpm +0.1rot 7.2dB 0e -0.8kHz gps4x7
+ICA484A9C>APRS,qAS,LFMX:/165341h4403.50N/00559.67E'/A=001460 id05484A9C +000fpm +0.0rot 18.0dB 0e +3.5kHz gps4x7
+OGNE95A16>APRS,qAS,Sylwek:/165641h5001.94N/01956.91E'270/004/A=000000 id07E95A16 +000fpm +0.1rot 37.8dB 0e -0.4kHz
+ZK-GSC>APRS,qAS,Omarama:/165202h4429.25S/16959.33E'/A=001407 id05C821EA +020fpm +0.0rot 16.8dB 0e -3.1kHz gps1x3 hear1084 hearB597 hearB598
+#
+# since 0.2.6 a aircraft beacon needs just an ID, climb rate and turn rate or just the ID
+ICA3ECE59>APRS,qAS,GLDRTR:/171254h5144.78N/00616.67E'263/000/A=000075 id093D0930 +000fpm +0.0rot
+ICA3ECE59>APRS,qAS,GLDRTR:/171254h5144.78N/00616.67E'263/000/A=000075 id053ECE59

tests/gateway/beacon_data/valid_beacon_data/aprs_receiver.txt

@@ -0,0 +1,30 @@
+# Until OGN software 0.2.6 all beacons (flarms, ogn trackers and receivers) have the dstcall "APRS"
+# These are example beacons for receivers
+#
+Lachens>APRS,TCPIP*,qAC,GLIDERN2:/165334h4344.70NI00639.19E&/A=005435 v0.2.1 CPU:0.3 RAM:1764.4/2121.4MB NTP:2.8ms/+4.9ppm +47.0C RF:+0.70dB
+LFGU>APRS,TCPIP*,qAC,GLIDERN2:/165556h4907.63NI00706.41E&/A=000833 v0.2.0 CPU:0.9 RAM:281.3/458.9MB NTP:0.5ms/-19.1ppm +53.0C RF:+0.70dB
+LSGS>APRS,TCPIP*,qAC,GLIDERN1:/165345h4613.25NI00719.68E&/A=001581 CPU:0.7 RAM:247.9/456.4MB NTP:0.7ms/-11.4ppm +44.4C RF:+53+71.9ppm/+0.4dB
+WolvesSW>APRS,TCPIP*,qAC,GLIDERN2:/165343h5232.23NI00210.91W&/A=000377 CPU:1.5 RAM:159.9/458.7MB NTP:6.6ms/-36.7ppm +45.5C RF:+130-0.4ppm/-0.1dB
+Oxford>APRS,TCPIP*,qAC,GLIDERN1:/165533h5142.96NI00109.68W&/A=000380 v0.1.3 CPU:0.9 RAM:268.8/458.6MB NTP:0.5ms/-45.9ppm +60.5C RF:+55+2.9ppm/+1.54dB
+Salland>APRS,TCPIP*,qAC,GLIDERN2:/165426h5227.93NI00620.03E&/A=000049 v0.2.2 CPU:0.7 RAM:659.3/916.9MB NTP:2.5ms/-75.0ppm RF:+0.41dB
+LSGS>APRS,TCPIP*,qAC,GLIDERN1:/165345h4613.25NI00719.68E&/A=001581 CPU:0.7 RAM:247.9/456.4MB NTP:0.7ms/-11.4ppm +44.4C RF:+53+71.9ppm/+0.4dB
+Drenstein>APRS,TCPIP*,qAC,GLIDERN1:/165011h5147.51NI00744.45E&/A=000213 v0.2.2 CPU:0.8 RAM:695.7/4025.5MB NTP:16000.0ms/+0.0ppm +63.0C
+#
+# since 0.2.5 for receiver information not only the "aprs position" format is used but also the "aprs status" format (without lat/lon/alt informations)
+Cordoba>APRS,TCPIP*,qAC,GLIDERN3:/194847h3112.85SI06409.56W&/A=001712 v0.2.5.ARM CPU:0.4 RAM:755.4/970.8MB NTP:6.7ms/-0.1ppm +45.5C RF:+48+18.3ppm/+3.45dB
+Cordoba>APRS,TCPIP*,qAC,GLIDERN3:>194847h v0.2.5.ARM CPU:0.4 RAM:755.4/970.8MB NTP:6.7ms/-0.1ppm +45.5C 0/0Acfts[1h] RF:+48+18.3ppm/+3.45dB/+0.4dB@10km[71]/+0.4dB@10km[1/1]
+VITACURA1>APRS,TCPIP*,qAC,GLIDERN3:/042149h3322.81SI07034.95W&/A=002345 v0.2.5.ARM CPU:0.6 RAM:694.4/970.5MB NTP:0.8ms/-7.5ppm +54.8C RF:+0-0.2ppm/+3.81dB
+VITACURA1>APRS,TCPIP*,qAC,GLIDERN3:>042149h v0.2.5.ARM CPU:0.6 RAM:694.4/970.5MB NTP:0.8ms/-7.5ppm +54.8C 0/0Acfts[1h] RF:+0-0.2ppm/+3.81dB/+1.3dB@10km[132205]/+6.6dB@10km[10/20]
+Arnsberg>APRS,TCPIP*,qAC,GLIDERN1:/042146h5123.04NI00803.77E&/A=000623 v0.2.5.ARM CPU:0.4 RAM:765.1/970.8MB NTP:0.4ms/-1.7ppm +62.3C RF:+27+1.1ppm/+3.17dB
+Arnsberg>APRS,TCPIP*,qAC,GLIDERN1:>042146h v0.2.5.ARM CPU:0.4 RAM:764.9/970.8MB NTP:0.4ms/-1.7ppm +62.3C 0/0Acfts[1h] RF:+27+1.1ppm/+3.17dB/+9.2dB@10km[44487]/+12.1dB@10km[20/40]
+CNF3a>APRS,TCPIP*,qAC,GLIDERN3:/042143h4529.25NI07505.65W&/A=000259 v0.2.5.ARM CPU:0.6 RAM:514.6/970.8MB NTP:4.5ms/-1.5ppm +27.2C RF:+0-0.4ppm/+18.69dB
+CNF3a>APRS,TCPIP*,qAC,GLIDERN3:>042143h v0.2.5.ARM CPU:0.6 RAM:514.6/970.8MB NTP:4.5ms/-1.5ppm +27.2C 0/0Acfts[1h] RF:+0-0.4ppm/+18.69dB/+13.0dB@10km[104282]/+9.7dB@10km[2/3]
+VITACURA2>APRS,TCPIP*,qAC,GLIDERN3:/042136h3322.81SI07034.95W&/A=002345 v0.2.5.ARM CPU:0.3 RAM:695.0/970.5MB NTP:0.6ms/-5.7ppm +51.5C RF:+0-0.0ppm/+1.32dB
+VITACURA2>APRS,TCPIP*,qAC,GLIDERN3:>042136h v0.2.5.ARM CPU:0.3 RAM:695.0/970.5MB NTP:0.6ms/-5.7ppm +52.1C 0/0Acfts[1h] RF:+0-0.0ppm/+1.32dB/+2.1dB@10km[193897]/+9.0dB@10km[10/20]
+#
+# since 0.2.6 the ogn comment of a receiver beacon is just optional, it also can be a user comment
+Ulrichamn>APRS,TCPIP*,qAC,GLIDERN1:/085616h5747.30NI01324.77E&/A=001322
+ROBLE3>APRS,TCPIP*,qAC,GLIDERN4:/200022h3258.58SI07100.78W&/A=007229 Contact: achanes@manquehue.net, brito.felipe@gmail.com
+#
+# ... and user comment can include a 'id'
+ALFALFAL>APRS,TCPIP*,qAC,GLIDERN4:/221830h3330.40SI07007.88W&/A=008659 Alfalfal Hidroelectric Plant, Club de Planeadores Vitacurs

tests/gateway/beacon_data/valid_beacon_data/capturs.txt

@@ -0,0 +1,11 @@
+# The following beacons are example for the Capture APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/062744h4845.03N/00230.46E'000/000/
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/064243h4839.64N/00236.78E'000/085/A=000410
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/064548h4838.87N/00234.03E'000/042/A=000377
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/064847h4837.95N/00234.36E'000/000/
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/065144h4837.56N/00233.80E'000/000/
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/065511h4837.63N/00233.79E'000/000/
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/070016h4837.63N/00233.77E'000/001/A=000360
+FLRDDEEF1>OGCAPT,qAS,CAPTURS:/070153h4837.62N/00233.77E'000/001/A=000344

tests/gateway/beacon_data/valid_beacon_data/fanet.txt

@@ -0,0 +1,8 @@
+# With OGN software 0.2.7 receivers have the dstcall "OGNFNT"
+#
+FNT1103CE>OGNFNT,qAS,FNB1103CE:/183727h5057.94N/00801.00Eg355/002/A=001042 !W10! id1E1103CE +03fpm
+FNT1103CE>OGNFNT,qAS,FNB1103CE:/183729h5057.94N/00801.00Eg354/001/A=001042 !W10! id1E1103CE +07fpm
+FNT1103CE>OGNFNT,qAS,FNB1103CE:/183731h5057.94N/00801.00Eg354/001/A=001042 !W10! id1E1103CE +05fpm
+FNT1103CE>OGNFNT,qAS,FNB1103CE:/183734h5057.94N/00801.00Eg354/001/A=001042 !W30! id1E1103CE -10fpm
+FNT1103CE>OGNFNT,qAS,FNB1103CE:/183736h5057.94N/00801.00Eg354/001/A=001042 !W40! id1E1103CE -02fpm
+FNB1103CE>OGNFNT,TCPIP*,qAC,GLIDERN3:/183738h5057.95NI00801.00E&/A=001042

tests/gateway/beacon_data/valid_beacon_data/flarm.txt

@@ -0,0 +1,7 @@
+# With OGN software 0.2.7 flarms have the dstcall "OGFLR"
+#
+FLRDD89C9>OGFLR,qAS,LIDH:/115054h4543.22N/01132.84E'260/072/A=002542 !W10! id06DD89C9 +198fpm -0.8rot 7.0dB 0e +0.7kHz gps2x3
+FLRDD98C6>OGFLR,qAS,LIDH:/115054h4543.21N/01132.80E'255/074/A=002535 !W83! id0ADD98C6 +158fpm -1.8rot 10.5dB 0e -0.8kHz gps2x3 s6.09 h02
+ICAA8CBA8>OGFLR,qAS,MontCAIO:/231150z4512.12N\01059.03E^192/106/A=009519 !W20! id21A8CBA8 -039fpm +0.0rot 3.5dB 2e -8.7kHz gps1x2 s6.09 h43 rDF0267
+ICAA8CBA8>OGFLR,qAS,MontCAIO:/114949h4512.44N\01059.12E^190/106/A=009522 !W33! id21A8CBA8 -039fpm +0.1rot 4.5dB 1e -8.7kHz gps1x2 +14.3dBm
+ICA3D1C35>OGFLR,qAS,Padova:/094220h4552.41N/01202.28E'110/099/A=003982 !W96! id053D1C35 -1187fpm +0.0rot 0.8dB 2e +4.5kHz gps1x2 s6.09 h32 rDD09D0

tests/gateway/beacon_data/valid_beacon_data/flymaster.txt

@@ -0,0 +1,35 @@
+# The following beacons are example for Flymaster APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+FMT924469>OGFLYM,qAS,FLYMASTER:/155232h3720.70N/00557.97W^222/092/A=000029 !W52!
+FMT003549>OGFLYM,qAS,FLYMASTER:/155231h3751.35N/00126.13W^270/022/A=001430 !W14!
+FMT001300>OGFLYM,qAS,FLYMASTER:/155249h3706.99N/00807.27W^178/000/A=000131 !W86!
+FMT798890>OGFLYM,qAS,FLYMASTER:/155256h3720.49N/00558.27W^234/086/A=000009 !W00!
+FMT549112>OGFLYM,qAS,FLYMASTER:/155256h3720.48N/00558.27W^234/086/A=000032 !W81!
+FMT148694>OGFLYM,qAS,FLYMASTER:/155244h3720.58N/00558.11W^226/087/A=000019 !W81!
+FMT842374>OGFLYM,qAS,FLYMASTER:/155302h3720.44N/00558.34W^236/082/A=000013 !W88!
+FMT003725>OGFLYM,qAS,FLYMASTER:/155304h3652.58N/00255.91W^346/000/A=001968 !W66!
+FMT924469>OGFLYM,qAS,FLYMASTER:/155306h3720.42N/00558.40W^250/081/A=000013 !W85!
+FMT003549>OGFLYM,qAS,FLYMASTER:/155316h3751.64N/00126.08W^020/048/A=001322 !W98!
+FMT148694>OGFLYM,qAS,FLYMASTER:/155318h3720.42N/00558.59W^282/088/A=000026 !W85!
+FMT549112>OGFLYM,qAS,FLYMASTER:/155328h3720.45N/00558.75W^282/079/A=000032 !W60!
+FMT842374>OGFLYM,qAS,FLYMASTER:/155335h3720.47N/00558.84W^280/078/A=000019 !W68!
+FMT001300>OGFLYM,qAS,FLYMASTER:/155339h3706.99N/00807.27W^178/000/A=000131 !W95!
+FMT798890>OGFLYM,qAS,FLYMASTER:/155338h3720.48N/00558.89W^282/080/A=000019 !W46!
+FMT924469>OGFLYM,qAS,FLYMASTER:/155341h3720.49N/00558.93W^282/075/A=000009 !W27!
+FMT003725>OGFLYM,qAS,FLYMASTER:/155346h3652.58N/00255.91W^346/000/A=001971 !W75!
+FMT003549>OGFLYM,qAS,FLYMASTER:/155349h3751.76N/00125.91W^064/032/A=001414 !W27!
+FMT148694>OGFLYM,qAS,FLYMASTER:/155352h3720.51N/00559.02W^292/026/A=000026 !W48!
+FMT549112>OGFLYM,qAS,FLYMASTER:/155400h3720.52N/00559.06W^298/031/A=000045 !W74!
+FMT842374>OGFLYM,qAS,FLYMASTER:/155409h3720.54N/00559.10W^302/019/A=000042 !W70!
+FMT798890>OGFLYM,qAS,FLYMASTER:/155412h3720.54N/00559.10W^304/001/A=000026 !W96!
+FMT924469>OGFLYM,qAS,FLYMASTER:/155415h3720.54N/00559.10W^000/001/A=000022 !W95!
+FMT003725>OGFLYM,qAS,FLYMASTER:/155420h3652.58N/00255.91W^346/000/A=001971 !W75!
+FMT003549>OGFLYM,qAS,FLYMASTER:/155422h3751.81N/00125.73W^220/002/A=001584 !W42!
+FMT001300>OGFLYM,qAS,FLYMASTER:/155429h3706.99N/00807.27W^178/000/A=000131 !W96!
+FMT148694>OGFLYM,qAS,FLYMASTER:/155435h3720.58N/00559.16W^314/017/A=000039 !W83!
+FMT549112>OGFLYM,qAS,FLYMASTER:/155443h3720.59N/00559.16W^000/000/A=000065 !W18!
+FMT798890>OGFLYM,qAS,FLYMASTER:/155444h3720.59N/00559.16W^000/000/A=000039 !W29!
+FMT924469>OGFLYM,qAS,FLYMASTER:/155447h3720.59N/00559.16W^000/000/A=000039 !W28!
+FMT842374>OGFLYM,qAS,FLYMASTER:/155453h3720.60N/00559.17W^316/020/A=000055 !W07!
+FMT003549>OGFLYM,qAS,FLYMASTER:/155455h3751.82N/00125.81W^248/012/A=001676 !W99!

tests/gateway/beacon_data/valid_beacon_data/inreach.txt

@@ -0,0 +1,4 @@
+# The following beacons are example for Garmin inReach APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+OGN8A0749>OGINRE,qAS,INREACH:/142700h0448.38N/07600.74W'000/000/A=004583 id300434060496190 inReac True

tests/gateway/beacon_data/valid_beacon_data/lt24.txt

@@ -0,0 +1,16 @@
+# The following beacons are example for LT24 APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+# the id25387 is the LT24 user ID that is unique.
+# GPS means that the coordinates are from the GPS unit of the smartphone vs. the GSM network.
+#
+FLRDDE48A>OGLT24,qAS,LT24:/102606h4030.47N/00338.38W'000/018/A=002267 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102608h4030.47N/00338.38W'044/018/A=002270 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102611h4030.47N/00338.38W'108/000/A=002280 id25387 +001fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102612h4030.47N/00338.38W'000/000/A=002280 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102615h4030.47N/00338.39W'224/003/A=002280 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102616h4030.47N/00338.38W'028/003/A=002250 id25387 -009fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102621h4030.47N/00338.38W'142/001/A=002267 id25387 +001fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102628h4030.47N/00338.38W'034/000/A=002263 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/102717h4030.47N/00338.38W'000/000/A=002263 id25387 +000fpm GPS
+FLRDDE48A>OGLT24,qAS,LT24:/110453h4030.47N/00338.38W'154/004/A=002253 id25387 +000fpm GPS

tests/gateway/beacon_data/valid_beacon_data/naviter.txt

@@ -0,0 +1,7 @@
+# The following beacons are example for NAVITER's APRS format version OGNAVI-1
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+NAV042121>OGNAVI,qAS,NAVITER:/140648h4550.36N/01314.85E'090/152/A=001086 !W47! id0440042121 +000fpm +0.5rot
+NAV04220E>OGNAVI,qAS,NAVITER:/140748h4552.27N/01155.61E'090/012/A=006562 !W81! id044004220E +060fpm +1.2rot
+NAV07220E>OGNAVI,qAS,NAVITER:/125447h4557.77N/01220.19E'258/056/A=006562 !W76! id1C4007220E +180fpm +0.0rot
+FLRFFFFFF>OGNAVI,NAV07220E*,qAS,NAVITER:/092002h1000.00S/01000.00W'000/000/A=003281 !W00! id2820FFFFFF +300fpm +1.7rot

tests/gateway/beacon_data/valid_beacon_data/pilot_aware.txt

@@ -0,0 +1,5 @@
+# The following beacons are example for PilotAware's APRS format version OGPAW-1
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+ICA404EC3>OGPAW,qAS,UKWOG:/104337h5211.24N\00032.65W^124/081/A=004026 !W62! id21404EC3 12.5dB +2.2kHz
+ICA404EC3>OGPAW,qAS,UKWOG:/104341h5211.18N\00032.53W^131/081/A=004010 !W85! id21404EC3 9.2dB +2.2kHz +10.0dBm

tests/gateway/beacon_data/valid_beacon_data/receiver.txt

@@ -0,0 +1,16 @@
+# With OGN software 0.2.7 receivers have the dstcall "OGNSDR"
+#
+LILH>OGNSDR,TCPIP*,qAC,GLIDERN2:/132201h4457.61NI00900.58E&/A=000423
+LILH>OGNSDR,TCPIP*,qAC,GLIDERN2:>132201h v0.2.7.RPI-GPU CPU:0.7 RAM:770.2/968.2MB NTP:1.8ms/-3.3ppm +55.7C 7/8Acfts[1h] RF:+54-1.1ppm/-0.16dB/+7.1dB@10km[19481]/+16.8dB@10km[7/13]
+MontCAIO>OGNSDR,TCPIP*,qAC,GLIDERN3:/132231h4427.84NI01009.60E&/A=004822
+MontCAIO>OGNSDR,TCPIP*,qAC,GLIDERN3:>132231h v0.2.7.RPI-GPU CPU:0.8 RAM:747.0/970.5MB NTP:2.8ms/-1.0ppm +73.1C 5/5Acfts[1h] RF:+69+1.3ppm/+3.53dB/+16.9dB@10km[7697]/+23.7dB@10km[3/6]
+Padova>OGNSDR,TCPIP*,qAC,GLIDERN1:/132326h4525.38NI01156.29E&/A=000069
+Padova>OGNSDR,TCPIP*,qAC,GLIDERN1:>132326h v0.2.7.RPI-GPU CPU:0.5 RAM:605.1/970.5MB NTP:0.5ms/-2.0ppm +65.5C 1/1Acfts[1h] RF:+0-1.1ppm/+13.97dB/+17.1dB@10km[6524]/+19.9dB@10km[5/9]
+LIDH>OGNSDR,TCPIP*,qAC,GLIDERN1:/132447h4540.89NI01129.65E&/A=000328
+LIDH>OGNSDR,TCPIP*,qAC,GLIDERN1:>132447h v0.2.7.RPI-GPU CPU:0.4 RAM:593.4/970.5MB NTP:3.7ms/-7.6ppm +67.7C 5/5Acfts[1h] RF:+61+1.0ppm/+12.63dB/+3.7dB@10km[27143]/+3.3dB@10km[3/6]
+LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:/132457h4849.09NI01708.30E&/A=000528
+LZHL>OGNSDR,TCPIP*,qAC,GLIDERN3:>132457h v0.2.7.arm CPU:0.9 RAM:75.3/253.6MB NTP:2.0ms/-15.2ppm +0.1C 2/2Acfts[1h] RF:+77+1.7ppm/+2.34dB/+6.5dB@10km[5411]/+10.1dB@10km[3/5]
+BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:/132507h4509.60NI00919.20E&/A=000246
+BELG>OGNSDR,TCPIP*,qAC,GLIDERN3:>132507h v0.2.7.RPI-GPU CPU:1.2 RAM:35.7/455.2MB NTP:2.5ms/-5.3ppm +67.0C 1/1Acfts[1h] RF:+79+8.8ppm/+4.97dB/-0.0dB@10km[299]/+4.9dB@10km[2/3]
+Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:/132624h4607.70NI00610.41E&/A=004198 Antenna: chinese, on a pylon, 20 meter above ground
+Saleve>OGNSDR,TCPIP*,qAC,GLIDERN1:>132624h v0.2.7.arm CPU:1.7 RAM:812.3/1022.5MB NTP:1.8ms/+4.5ppm 0.000V 0.000A 3/4Acfts[1h] RF:+67+2.9ppm/+4.18dB/+11.7dB@10km[5018]/+17.2dB@10km[8/16]

tests/gateway/beacon_data/valid_beacon_data/skylines.txt

@@ -0,0 +1,6 @@
+# The following beacons are example for Skylines (XCsoar) APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+# The id2816 is the Pilot ID from Skylines (XCsoar) and it is unique within the skyline system
+#
+FLRDDDD78>OGSKYL,qAS,SKYLINES:/134403h4225.90N/00144.83E'000/000/A=008438 id2816 +000fpm

tests/gateway/beacon_data/valid_beacon_data/spider.txt

@@ -0,0 +1,23 @@
+# The following beacons are example for Spider APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+# id3003... is the unique identifier from the SPIDER server
+# LWE is the registration within the SPIDER system
+# 3D is the quality of the signal  3D vs. 2D
+#
+FLRDDF944>OGSPID,qAS,SPIDER:/190930h3322.78S/07034.60W'000/000/A=002263 id300234010617040 +19dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/190930h3322.78S/07034.60W'000/000/A=002263 id300234010617040 +19dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/192430h3322.78S/07034.61W'000/000/A=002250 id300234010617040 +12dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/193930h3322.10S/07034.26W'273/027/A=004071 id300234010617040 +9dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/195430h3322.82S/07034.90W'000/000/A=002217 id300234010617040 +10dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/193930h3322.78S/07034.60W'348/000/A=002286 id300234010617040 +12dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/195430h3323.16S/07037.68W'302/034/A=003316 id300234010617040 +10dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/195430h3323.16S/07037.68W'302/034/A=003316 id300234010617040 +10dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/200930h3319.13S/07036.12W'128/031/A=005482 id300234010617040 +15dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/200930h3319.13S/07036.12W'128/031/A=005482 id300234010617040 +15dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/202430h3314.92S/07032.08W'138/032/A=006453 id300234010617040 +9dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/203930h3321.38S/07027.29W'104/034/A=006272 id300234010617040 +8dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/205430h3322.13S/07033.53W'296/031/A=003927 id300234010617040 +7dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/210930h3322.05S/07035.74W'165/030/A=005187 id300234010617040 +8dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/212430h3322.02S/07036.14W'281/028/A=004550 id300234010617040 +7dB LWE 3D
+FLRDDF944>OGSPID,qAS,SPIDER:/213930h3322.17S/07033.97W'332/028/A=003428 id300234010617040 +7dB LWE 3D

tests/gateway/beacon_data/valid_beacon_data/spot.txt

@@ -0,0 +1,11 @@
+# The following beacons are example for SPOT APRS format
+# source: https://github.com/glidernet/ogn-aprs-protocol
+#
+# id0-28... is a unique ID within the SPOT system.
+# SPOT2 is the Spot model
+# GOOD is the battery status or some other help messages.
+#
+ICA3E7540>OGSPOT,qAS,SPOT:/161427h1448.35S/04610.86W'000/000/A=008677 id0-2860357 SPOT3 GOOD
+ICA3E7540>OGSPOT,qAS,SPOT:/162923h1431.99S/04604.33W'000/000/A=006797 id0-2860357 SPOT3 GOOD
+ICA3E7540>OGSPOT,qAS,SPOT:/163421h1430.38S/04604.43W'000/000/A=007693 id0-2860357 SPOT3 GOOD
+FLRDF0CBA>OGSPOT,qAS,SPOT:/145808h3317.84S/07021.04W'000/000/A=010085 id0-2120121 SPOTCONNECT GOOD

tests/gateway/beacon_data/valid_beacon_data/tracker.txt

@@ -0,0 +1,11 @@
+# With OGN software 0.2.7 ogn tracker have the dstcall "OGNTRK"
+#
+OGN3FC859>OGNTRK,qAS,LZHL:>093215h h00 v00 9sat/1 164m 1002.6hPa +20.2degC 0% 3.34V 14/-110.5dBm 1/min
+OGN2FD00F>OGNTRK,qAS,LZHL:/093213h4848.78N/01708.32E'000/000/A=000538 !W12! id072FD00F -058fpm +0.0rot FL003.12 32.8dB 0e -0.8kHz gps3x5
+FLRDD9C70>OGNTRK,OGN2FD00F*,qAS,LZHL:/093214h4848.77N/01708.33E'000/000/A=000515 !W56! id06DD9C70 -019fpm +0.0rot 32.2dB 0e -0.8kHz gps2x3
+FLRDD9C70>OGNTRK,OGN2FD00F*,qAS,LZHL:/093021h4848.77N/01708.33E'000/000/A=000518 !W66! id06DD9C70 -019fpm +0.0rot 29.0dB 0e -0.8kHz gps2x3 s6.09 h03
+OGN03AF2A>OGNTRK,qAS,LZHL:/092912h4848.77N/01708.33E'000/000/A=000535 !W53! id0703AF2A +000fpm +0.0rot FL003.15 4.5dB 1e -0.1kHz gps4x5 -11.2dBm
+OGN2FD00F>OGNTRK,qAS,LZHL:>092840h h00 v00 11sat/2 165m 1001.9hPa +27.1degC 0% 3.28V 14/-111.5dBm 127/min
+FLRDD9C70>OGNTRK,RELAY*,qAS,LZHL:/094124h4848.78N/01708.33E'000/000/A=000397 !W15! id06DD9C70 +099fpm +0.0rot 24.5dB 0e -1.4kHz gps10x15
+OGN7402C8>OGNTRK,qAS,OxfBarton:>055357h h02 v01
+OGN395F39>OGNTRK,qAS,OxfBarton:>055451h Pilot=Pawel Hard=DIY/STM32

tests/gateway/test_bulkimport.py

@@ -1,6 +1,6 @@
 import os
 import unittest
-from datetime import datetime
+import datetime
 from app.model import AircraftBeacon, ReceiverBeacon
 from app.gateway.bulkimport import DbFeeder
 
@@ -12,17 +12,15 @@ class TestDatabase(TestBaseDB):
         """This test insert all valid beacons. source: https://github.com/glidernet/ogn-aprs-protocol/valid_messages"""
 
         path = os.path.join(os.path.dirname(__file__), 'valid_messages')
-        with DbFeeder(reference_timestamp=datetime.utcnow(), reference_timestamp_autoupdate=True) as feeder:
+        with os.scandir(path) as it:
-            with os.scandir(path) as it:
+            for entry in it:
-                for entry in it:
+                if entry.name.endswith(".txt") and entry.is_file():
-                    if entry.name.endswith(".txt") and entry.is_file():
+                    with DbFeeder() as feeder:
                         print(f"Parsing {entry.name}")
                         with open(entry.path) as file:
                             for line in file:
-                                feeder.add(line)
+                                feeder.add(line, datetime.datetime(2020, 5, 1, 13, 22, 1))
-                        feeder.flush()
 
-    @unittest.skip('currently only positions are considered')
     def test_ognsdr_beacons(self):
         """This test tests if status+position is correctly merged."""
 
@@ -31,14 +29,21 @@ class TestDatabase(TestBaseDB):
             "LILH>OGNSDR,TCPIP*,qAC,GLIDERN2:>132201h v0.2.7.RPI-GPU CPU:0.7 RAM:770.2/968.2MB NTP:1.8ms/-3.3ppm +55.7C 7/8Acfts[1h] RF:+54-1.1ppm/-0.16dB/+7.1dB@10km[19481]/+16.8dB@10km[7/13]"
         )
 
-        with DbFeeder(reference_timestamp=datetime.utcnow(), reference_timestamp_autoupdate=True) as feeder:
+        with DbFeeder() as feeder:
             for line in aprs_stream.split('\n'):
-                feeder.add(line)
+                feeder.add(line, datetime.datetime(2020, 5, 1, 13, 22, 1))
 
         self.assertEqual(len(db.session.query(ReceiverBeacon).all()), 1)
         for ab in db.session.query(ReceiverBeacon).all():
             print(ab)
 
+    def test_oneminute(self):
+        with DbFeeder() as feeder:
+            with open(os.path.dirname(__file__) + '/oneminute.txt') as f:
+                for line in f:
+                    timestamp = datetime.datetime.strptime(line[:26], '%Y-%m-%d %H:%M:%S.%f')
+                    aprs_string = line[28:]
+                    feeder.add(aprs_string, reference_timestamp=timestamp)
 
 if __name__ == "__main__":
     unittest.main()