From 712a067d706916d082090e54c6f77500fc6dd276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Gru=CC=88ndger?= Date: Mon, 25 Feb 2019 20:00:51 +0100 Subject: [PATCH] Click integration --- ogn_python/__init__.py | 5 ++ ogn_python/commands/__init__.py | 3 + ogn_python/commands/database.py | 13 ++++- ogn_python/commands/gateway.py | 34 +++++++++++ ogn_python/gateway/manage.py | 55 ------------------ ogn_python/gateway/process.py | 71 ----------------------- ogn_python/gateway/process_tools.py | 87 ++++++++++++++++++++++------- 7 files changed, 120 insertions(+), 148 deletions(-) create mode 100644 ogn_python/commands/gateway.py delete mode 100644 ogn_python/gateway/manage.py delete mode 100644 ogn_python/gateway/process.py diff --git a/ogn_python/__init__.py b/ogn_python/__init__.py index 3c80eec..8478c04 100644 --- a/ogn_python/__init__.py +++ b/ogn_python/__init__.py @@ -1,13 +1,18 @@ from flask import Flask from flask_bootstrap import Bootstrap from flask_sqlalchemy import SQLAlchemy + from ogn_python.navigation import nav +# Initialize Flask app = Flask(__name__) app.config.from_object('config.default') +# Bootstrap bootstrap = Bootstrap(app) +# Sqlalchemy db = SQLAlchemy(app) +# Navigation nav.init_app(app) diff --git a/ogn_python/commands/__init__.py b/ogn_python/commands/__init__.py index f10e0b0..d126d6e 100644 --- a/ogn_python/commands/__init__.py +++ b/ogn_python/commands/__init__.py @@ -4,11 +4,14 @@ from .bulkimport import user_cli as bulkimport_cli from .database import user_cli as database_cli from .export import user_cli as export_cli from .flights import user_cli as flights_cli +from .gateway import user_cli as gateway_cli from .logbook import user_cli as logbook_cli from .stats import user_cli as stats_cli app.cli.add_command(bulkimport_cli) app.cli.add_command(database_cli) +app.cli.add_command(export_cli) app.cli.add_command(flights_cli) +app.cli.add_command(gateway_cli) app.cli.add_command(logbook_cli) app.cli.add_command(stats_cli) diff --git a/ogn_python/commands/database.py b/ogn_python/commands/database.py index 4b1b723..8fa3340 100644 --- a/ogn_python/commands/database.py +++ b/ogn_python/commands/database.py @@ -32,6 +32,15 @@ def get_database_days(start, end): return days +@user_cli.command('info') +def info(): + import importlib + import os + config = importlib.import_module(os.environ['OGN_CONFIG_MODULE']) + print(config) + print(config.SQLALCHEMY_DATABASE_URI) + + @user_cli.command('init') def init(): """Initialize the database.""" @@ -71,8 +80,8 @@ def upgrade(): @user_cli.command('drop') -@click.argument('sure') -def drop(sure='n'): +@click.option('--sure', default='n') +def drop(sure): """Drop all tables.""" if sure == 'y': Base.metadata.drop_all(engine) diff --git a/ogn_python/commands/gateway.py b/ogn_python/commands/gateway.py new file mode 100644 index 0000000..e553c53 --- /dev/null +++ b/ogn_python/commands/gateway.py @@ -0,0 +1,34 @@ +from flask.cli import AppGroup +import click + +from ogn.client import AprsClient +from ogn_python.gateway.process_tools import DbSaver + +from ogn_python import db + +user_cli = AppGroup('gateway') +user_cli.help = "Connection to APRS servers." + + +@user_cli.command('run') +def run(aprs_user='anon-dev'): + """Run the aprs client.""" + + saver = DbSaver(session=db.session) + + # User input validation + if len(aprs_user) < 3 or len(aprs_user) > 9: + print('aprs_user must be a string of 3-9 characters.') + return + + print('Start ogn gateway') + client = AprsClient(aprs_user) + client.connect() + + try: + client.run(callback=saver.add_raw_message, autoreconnect=True) + except KeyboardInterrupt: + print('\nStop ogn gateway') + + saver.flush() + client.disconnect() diff --git a/ogn_python/gateway/manage.py b/ogn_python/gateway/manage.py deleted file mode 100644 index a388af8..0000000 --- a/ogn_python/gateway/manage.py +++ /dev/null @@ -1,55 +0,0 @@ -import logging - -from manager import Manager -from ogn_python.client import AprsClient -from ogn_python.gateway.process import string_to_message -from datetime import datetime -from ogn_python.gateway.process_tools import DbSaver -from ogn_python.commands.dbutils import session - -manager = Manager() - -logging_formatstr = '%(asctime)s - %(levelname).4s - %(name)s - %(message)s' -log_levels = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'] - -saver = DbSaver(session=session) - - -def asdf(raw_string): - message = string_to_message(raw_string, reference_date=datetime.utcnow()) - if message is not None: - saver.add_message(message) - else: - print(message) - - -@manager.command -def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): - """Run the aprs client.""" - - # User input validation - if len(aprs_user) < 3 or len(aprs_user) > 9: - print('aprs_user must be a string of 3-9 characters.') - return - if loglevel not in log_levels: - print('loglevel must be an element of {}.'.format(log_levels)) - return - - # Enable logging - log_handlers = [logging.StreamHandler()] - if logfile: - log_handlers.append(logging.FileHandler(logfile)) - logging.basicConfig(format=logging_formatstr, level=loglevel, handlers=log_handlers) - - print('Start ogn gateway') - client = AprsClient(aprs_user) - client.connect() - - try: - client.run(callback=asdf, autoreconnect=True) - except KeyboardInterrupt: - print('\nStop ogn gateway') - - saver.flush() - client.disconnect() - logging.shutdown() diff --git a/ogn_python/gateway/process.py b/ogn_python/gateway/process.py deleted file mode 100644 index 68e4ff7..0000000 --- a/ogn_python/gateway/process.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging - -from mgrs import MGRS - -from ogn_python.commands.dbutils import session -from ogn_python.model import Location -from ogn_python.parser import parse, ParseError -from ogn_python.gateway.process_tools import DbSaver, AIRCRAFT_BEACON_TYPES, RECEIVER_BEACON_TYPES - - -logger = logging.getLogger(__name__) -myMGRS = MGRS() - - -def _replace_lonlat_with_wkt(message): - latitude = message['latitude'] - longitude = message['longitude'] - - location = Location(longitude, latitude) - message['location_wkt'] = location.to_wkt() - location_mgrs = myMGRS.toMGRS(latitude, longitude).decode('utf-8') - message['location_mgrs'] = location_mgrs - message['location_mgrs_short'] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] - del message['latitude'] - del message['longitude'] - return message - - -def string_to_message(raw_string, reference_date): - global receivers - - try: - message = parse(raw_string, reference_date) - except NotImplementedError as e: - logger.w('No parser implemented for message: {}'.format(raw_string)) - return None - except ParseError as e: - logger.error('Parsing error with message: {}'.format(raw_string)) - return None - except TypeError as e: - logger.error('TypeError with message: {}'.format(raw_string)) - return None - except Exception as e: - logger.error(raw_string) - logger.error(e) - return None - - # update reference receivers and distance to the receiver - if message['aprs_type'] == 'position': - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES + RECEIVER_BEACON_TYPES: - message = _replace_lonlat_with_wkt(message) - - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and 'gps_quality' in message: - if message['gps_quality'] is not None and 'horizontal' in message['gps_quality']: - message['gps_quality_horizontal'] = message['gps_quality']['horizontal'] - message['gps_quality_vertical'] = message['gps_quality']['vertical'] - del message['gps_quality'] - - # update raw_message - message['raw_message'] = raw_string - - return message - - -saver = DbSaver(session=session) - - -def process_raw_message(raw_message, reference_date=None, saver=saver): - logger.debug('Received message: {}'.format(raw_message)) - message = string_to_message(raw_message, reference_date) - saver.add_message(message) diff --git a/ogn_python/gateway/process_tools.py b/ogn_python/gateway/process_tools.py index 54098f1..3971a36 100644 --- a/ogn_python/gateway/process_tools.py +++ b/ogn_python/gateway/process_tools.py @@ -1,7 +1,11 @@ from datetime import datetime, timedelta -from ogn_python.model import AircraftBeacon, ReceiverBeacon +from ogn.parser import parse, ParseError +from ogn_python.model import AircraftBeacon, ReceiverBeacon, Location from ogn_python.collect.database import upsert +from mgrs import MGRS + + # define message types we want to proceed AIRCRAFT_BEACON_TYPES = ['aprs_aircraft', 'flarm', 'tracker', 'fanet', 'lt24', 'naviter', 'skylines', 'spider', 'spot'] RECEIVER_BEACON_TYPES = ['aprs_receiver', 'receiver'] @@ -12,15 +16,57 @@ AIRCRAFT_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'track', ' RECEIVER_BEACON_FIELDS = ['location', 'altitude', 'dstcall', 'relay', 'version', 'platform', 'cpu_load', 'free_ram', 'total_ram', 'ntp_error', 'rt_crystal_correction', 'voltage', 'amperage', 'cpu_temp', 'senders_visible', 'senders_total', 'rec_input_noise', 'senders_signal', 'senders_messages', 'good_senders_signal', 'good_senders', 'good_and_bad_senders'] -class DummyMerger: - def __init__(self, callback): - self.callback = callback +myMGRS = MGRS() - def add_message(self, message): - self.callback.add_message(message) - def flush(self): - pass +def _replace_lonlat_with_wkt(message): + latitude = message['latitude'] + longitude = message['longitude'] + + location = Location(longitude, latitude) + message['location_wkt'] = location.to_wkt() + location_mgrs = myMGRS.toMGRS(latitude, longitude).decode('utf-8') + message['location_mgrs'] = location_mgrs + message['location_mgrs_short'] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] + del message['latitude'] + del message['longitude'] + return message + + +def string_to_message(raw_string, reference_date): + global receivers + + try: + message = parse(raw_string, reference_date) + except NotImplementedError as e: + print('No parser implemented for message: {}'.format(raw_string)) + return None + except ParseError as e: + print('Parsing error with message: {}'.format(raw_string)) + return None + except TypeError as e: + print('TypeError with message: {}'.format(raw_string)) + return None + except Exception as e: + print(raw_string) + print(e) + return None + + # update reference receivers and distance to the receiver + if message['aprs_type'] == 'position': + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES + RECEIVER_BEACON_TYPES: + message = _replace_lonlat_with_wkt(message) + + if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and 'gps_quality' in message: + if message['gps_quality'] is not None and 'horizontal' in message['gps_quality']: + message['gps_quality_horizontal'] = message['gps_quality']['horizontal'] + message['gps_quality_vertical'] = message['gps_quality']['vertical'] + del message['gps_quality'] + + # update raw_message + message['raw_message'] = raw_string + + return message class DbSaver: @@ -40,6 +86,15 @@ class DbSaver: else: my_map[key] = message + def add_raw_message(self, raw_string, reference_date=None): + if not reference_date: + reference_date=datetime.utcnow() + message = string_to_message(raw_string, reference_date=reference_date) + if message is not None: + self.add_message(message) + else: + print(raw_string) + def add_message(self, message): if message is None or ('raw_message' in message and message['raw_message'][0] == '#') or 'beacon_type' not in message: return @@ -48,11 +103,11 @@ class DbSaver: message['location'] = message.pop('location_wkt') # total_time_wasted_here = 3 if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: - even_messages = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS} - self._put_in_map(message=even_messages, my_map=self.aircraft_message_map) + complete_message = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + AIRCRAFT_BEACON_FIELDS} + self._put_in_map(message=complete_message, my_map=self.aircraft_message_map) elif message['beacon_type'] in RECEIVER_BEACON_TYPES: - even_messages = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS} - self._put_in_map(message=even_messages, my_map=self.receiver_message_map) + complete_message = {k: message[k] if k in message else None for k in BEACON_KEY_FIELDS + RECEIVER_BEACON_FIELDS} + self._put_in_map(message=complete_message, my_map=self.receiver_message_map) else: print("Ignore beacon_type: {}".format(message['beacon_type'])) return @@ -75,14 +130,6 @@ class DbSaver: self.last_commit = datetime.utcnow() -class DummySaver: - def add_message(self, message): - print(message) - - def flush(self): - print("========== flush ==========") - - import os, gzip, csv