diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f0a13..e0c7c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## Unreleased +- Moved exceptions from `ogn.exceptions` to `ogn.parser.exceptions` +- Moved parsing from `ogn.model.*` to `ogn.parser` + ## 0.2.1 - 2016-02-17 First release via PyPi. - Added CHANGELOG. diff --git a/README.md b/README.md index 75a76c8..2a55970 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,23 @@ lets you process the incoming data. Example: ```python #!/usr/bin/env python3 - -from ogn.model import AircraftBeacon, ReceiverBeacon from ogn.gateway.client import ognGateway +from ogn.parser.parse import parse_aprs, parse_ogn_beacon +from ogn.parser.exceptions import ParseError -def process_beacon(beacon): - if type(beacon) is AircraftBeacon: - print('Received aircraft beacon from {}'.format(beacon.name)) - elif type(beacon) is ReceiverBeacon: - print('Received receiver beacon from {}'.format(beacon.name)) +def process_beacon(raw_message): + if raw_message[0] == '#': + print('Server Status: {}'.format(raw_message)) + return + + try: + message = parse_aprs(raw_message) + message.update(parse_ogn_beacon(message['comment'])) + + print('Received {beacon_type} from {name}'.format(**message)) + except ParseError as e: + print('Error, {}'.format(e.message)) if __name__ == '__main__': diff --git a/ogn/aprs_parser.py b/ogn/aprs_parser.py deleted file mode 100644 index 8ebd7a7..0000000 --- a/ogn/aprs_parser.py +++ /dev/null @@ -1,35 +0,0 @@ -from datetime import datetime - -from .model import Beacon, AircraftBeacon, ReceiverBeacon -from ogn.exceptions import AprsParseError - - -def parse_aprs(packet, reference_date=None): - if reference_date is None: - reference_date = datetime.utcnow() - if not isinstance(packet, str): - raise TypeError("Expected packet to be str, got %s" % type(packet)) - elif packet == "": - raise AprsParseError("(empty string)") - elif packet[0] == "#": - return None - - beacon = Beacon() - beacon.parse(packet, reference_date) - - # symboltable / symbolcodes used by OGN: - # I&: used as receiver - # /X: helicopter_rotorcraft - # /': glider_or_motorglider - # \^: powered_aircraft - # /g: para_glider - # /O: ? - # /^: ? - # \n: ? - # /z: ? - # /o: ? - - if beacon.symboltable == "I" and beacon.symbolcode == "&": - return ReceiverBeacon(beacon) - else: - return AircraftBeacon(beacon) diff --git a/ogn/gateway/client.py b/ogn/gateway/client.py index cb8d806..69ab8ce 100644 --- a/ogn/gateway/client.py +++ b/ogn/gateway/client.py @@ -3,9 +3,13 @@ import logging from time import time from ogn.gateway import settings -from ogn.aprs_parser import parse_aprs -from ogn.aprs_utils import create_aprs_login -from ogn.exceptions import AprsParseError, OgnParseError, AmbigousTimeError + + +def create_aprs_login(user_name, pass_code, app_name, app_version, aprs_filter=None): + if not aprs_filter: + return "user {} pass {} vers {} {}\n".format(user_name, pass_code, app_name, app_version) + else: + return "user {} pass {} vers {} {} filter {}\n".format(user_name, pass_code, app_name, app_version, aprs_filter) class ognGateway: @@ -42,8 +46,6 @@ class ognGateway: self.logger.error('Socket close error', exc_info=True) def run(self, callback, autoreconnect=False): - self.process_beacon = callback - while True: try: keepalive_time = time() @@ -62,7 +64,7 @@ class ognGateway: self.logger.warning('Read returns zero length string. Failure. Orderly closeout') break - self.proceed_line(packet_str) + callback(packet_str) except BrokenPipeError: self.logger.error('BrokenPipeError', exc_info=True) except socket.error: @@ -72,20 +74,3 @@ class ognGateway: self.connect() else: return - - def proceed_line(self, line): - try: - beacon = parse_aprs(line) - self.logger.debug('Received beacon: {}'.format(beacon)) - except AprsParseError: - self.logger.error('AprsParseError while parsing line: {}'.format(line), exc_info=True) - return - except OgnParseError: - self.logger.error('OgnParseError while parsing line: {}'.format(line), exc_info=True) - return - except AmbigousTimeError as e: - self.logger.error('Drop packet, {:.0f}s from past: {}'.format(e.timedelta.total_seconds(), line)) - return - - if beacon is not None: - self.process_beacon(beacon) diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index b4804da..e72a00c 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -1,7 +1,7 @@ import logging from ogn.gateway.client import ognGateway -from ogn.commands.dbutils import session +from ogn.gateway.process import process_beacon from manager import Manager @@ -33,10 +33,6 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): gateway = ognGateway(aprs_user) gateway.connect() - def process_beacon(beacon): - session.add(beacon) - session.commit() - try: gateway.run(callback=process_beacon, autoreconnect=True) except KeyboardInterrupt: diff --git a/ogn/gateway/process.py b/ogn/gateway/process.py new file mode 100644 index 0000000..145cc7e --- /dev/null +++ b/ogn/gateway/process.py @@ -0,0 +1,38 @@ +import logging +from ogn.commands.dbutils import session +from ogn.model import AircraftBeacon, ReceiverBeacon +from ogn.parser.parse import parse_aprs, parse_ogn_receiver_beacon, parse_ogn_aircraft_beacon +from ogn.parser.exceptions import AprsParseError, OgnParseError, AmbigousTimeError + +logger = logging.getLogger(__name__) + + +def process_beacon(raw_message): + if raw_message[0] == '#': + return + try: + message = parse_aprs(raw_message) + + # symboltable / symbolcodes used by OGN: + # I&: used as receiver + # /X: helicopter_rotorcraft + # /': glider_or_motorglider + # \^: powered_aircraft + # /g: para_glider + # /O: ? + # /^: ? + # \n: ? + # /z: ? + # /o: ? + if message['symboltable'] == "I" and message['symbolcode'] == '&': + message.update(parse_ogn_receiver_beacon(message['comment'])) + beacon = ReceiverBeacon(**message) + else: + message.update(parse_ogn_aircraft_beacon(message['comment'])) + beacon = AircraftBeacon(**message) + session.add(beacon) + session.commit() + logger.debug('Received message: {}'.format(raw_message)) + except (AprsParseError, OgnParseError, AmbigousTimeError) as e: + logger.error('Received message: {}'.format(raw_message)) + logger.error('Drop packet, {}'.format(e.message)) diff --git a/ogn/model/aircraft_beacon.py b/ogn/model/aircraft_beacon.py index 352c1f3..02677ef 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn/model/aircraft_beacon.py @@ -1,10 +1,6 @@ -import re - from sqlalchemy import Column, String, Integer, Float, Boolean, SmallInteger -from ogn.aprs_utils import fpm2ms from .beacon import Beacon -from ogn.exceptions import OgnParseError class AircraftBeacon(Beacon): @@ -35,103 +31,6 @@ class AircraftBeacon(Beacon): flight_state = Column(SmallInteger) - # Pattern - address_pattern = re.compile(r"id(\S{2})(\S{6})") - climb_rate_pattern = re.compile(r"([\+\-]\d+)fpm") - turn_rate_pattern = re.compile(r"([\+\-]\d+\.\d+)rot") - signal_strength_pattern = re.compile(r"(\d+\.\d+)dB") - error_count_pattern = re.compile(r"(\d+)e") - coordinates_extension_pattern = re.compile(r"\!W(.)(.)!") - hear_address_pattern = re.compile(r"hear(\w{4})") - frequency_offset_pattern = re.compile(r"([\+\-]\d+\.\d+)kHz") - gps_status_pattern = re.compile(r"gps(\d+x\d+)") - - software_version_pattern = re.compile(r"s(\d+\.\d+)") - hardware_version_pattern = re.compile(r"h(\d+)") - real_address_pattern = re.compile(r"r(\w{6})") - - flightlevel_pattern = re.compile(r"FL(\d{3}\.\d{2})") - - def __init__(self, beacon=None): - self.heared_aircraft_addresses = list() - - if beacon is not None: - self.name = beacon.name - self.receiver_name = beacon.receiver_name - self.timestamp = beacon.timestamp - self.latitude = beacon.latitude - self.longitude = beacon.longitude - self.ground_speed = beacon.ground_speed - self.track = beacon.track - self.altitude = beacon.altitude - self.comment = beacon.comment - - self.parse(beacon.comment) - else: - self.latitude = 0.0 - self.longitude = 0.0 - - def parse(self, text): - for part in text.split(' '): - address_match = self.address_pattern.match(part) - climb_rate_match = self.climb_rate_pattern.match(part) - turn_rate_match = self.turn_rate_pattern.match(part) - signal_strength_match = self.signal_strength_pattern.match(part) - error_count_match = self.error_count_pattern.match(part) - coordinates_extension_match = self.coordinates_extension_pattern.match(part) - hear_address_match = self.hear_address_pattern.match(part) - frequency_offset_match = self.frequency_offset_pattern.match(part) - gps_status_match = self.gps_status_pattern.match(part) - - software_version_match = self.software_version_pattern.match(part) - hardware_version_match = self.hardware_version_pattern.match(part) - real_address_match = self.real_address_pattern.match(part) - - flightlevel_match = self.flightlevel_pattern.match(part) - - if address_match is not None: - # Flarm ID type byte in APRS msg: PTTT TTII - # P => stealth mode - # TTTTT => aircraftType - # II => IdType: 0=Random, 1=ICAO, 2=FLARM, 3=OGN - # (see https://groups.google.com/forum/#!msg/openglidernetwork/lMzl5ZsaCVs/YirmlnkaJOYJ). - self.address_type = int(address_match.group(1), 16) & 0b00000011 - self.aircraft_type = (int(address_match.group(1), 16) & 0b01111100) >> 2 - self.stealth = ((int(address_match.group(1), 16) & 0b10000000) >> 7 == 1) - self.address = address_match.group(2) - elif climb_rate_match is not None: - self.climb_rate = int(climb_rate_match.group(1)) * fpm2ms - elif turn_rate_match is not None: - self.turn_rate = float(turn_rate_match.group(1)) - elif signal_strength_match is not None: - self.signal_strength = float(signal_strength_match.group(1)) - elif error_count_match is not None: - self.error_count = int(error_count_match.group(1)) - elif coordinates_extension_match is not None: - dlat = int(coordinates_extension_match.group(1)) / 1000 / 60 - dlon = int(coordinates_extension_match.group(2)) / 1000 / 60 - - self.latitude = self.latitude + dlat - self.longitude = self.longitude + dlon - elif hear_address_match is not None: - self.heared_aircraft_addresses.append(hear_address_match.group(1)) - elif frequency_offset_match is not None: - self.frequency_offset = float(frequency_offset_match.group(1)) - elif gps_status_match is not None: - self.gps_status = gps_status_match.group(1) - - elif software_version_match is not None: - self.software_version = float(software_version_match.group(1)) - elif hardware_version_match is not None: - self.hardware_version = int(hardware_version_match.group(1)) - elif real_address_match is not None: - self.real_address = real_address_match.group(1) - - elif flightlevel_match is not None: - self.flightlevel = float(flightlevel_match.group(1)) - else: - raise OgnParseError(expected_type="AircraftBeacon", substring=part) - def __repr__(self): return "" % ( self.name, diff --git a/ogn/model/beacon.py b/ogn/model/beacon.py index fd00df5..8a7503d 100644 --- a/ogn/model/beacon.py +++ b/ogn/model/beacon.py @@ -1,19 +1,9 @@ -import re -from datetime import datetime - from sqlalchemy import Column, String, Integer, Float, DateTime from sqlalchemy.ext.declarative import AbstractConcreteBase -from ogn.aprs_utils import createTimestamp, dmsToDeg, kts2kmh, feet2m -from ogn.exceptions import AprsParseError from .base import Base -# "original" pattern from OGN: "(.+?)>APRS,.+,(.+?):/(\\d{6})+h(\\d{4}\\.\\d{2})(N|S).(\\d{5}\\.\\d{2})(E|W).((\\d{3})/(\\d{3}))?/A=(\\d{6}).*?" -PATTERN_APRS = r"^(.+?)>APRS,.+,(.+?):/(\d{6})+h(\d{4}\.\d{2})(N|S)(.)(\d{5}\.\d{2})(E|W)(.)((\d{3})/(\d{3}))?/A=(\d{6})\s(.*)$" -re_pattern_aprs = re.compile(PATTERN_APRS) - - class Beacon(AbstractConcreteBase, Base): id = Column(Integer, primary_key=True) @@ -29,38 +19,3 @@ class Beacon(AbstractConcreteBase, Base): ground_speed = Column(Float) altitude = Column(Integer) comment = None - - def parse(self, text, reference_date=None): - if reference_date is None: - reference_date = datetime.utcnow() - result = re_pattern_aprs.match(text) - if result is None: - raise AprsParseError(text) - - self.name = result.group(1) - self.receiver_name = result.group(2) - - self.timestamp = createTimestamp(result.group(3), reference_date) - - self.latitude = dmsToDeg(float(result.group(4)) / 100) - if result.group(5) == "S": - self.latitude = -self.latitude - - self.symboltable = result.group(6) - - self.longitude = dmsToDeg(float(result.group(7)) / 100) - if result.group(8) == "W": - self.longitude = -self.longitude - - self.symbolcode = result.group(9) - - if result.group(10) is not None: - self.track = int(result.group(11)) - self.ground_speed = int(result.group(12)) * kts2kmh - else: - self.track = 0 - self.ground_speed = 0 - - self.altitude = int(result.group(13)) * feet2m - - self.comment = result.group(14) diff --git a/ogn/model/receiver_beacon.py b/ogn/model/receiver_beacon.py index d566d51..1b6a3f2 100644 --- a/ogn/model/receiver_beacon.py +++ b/ogn/model/receiver_beacon.py @@ -1,9 +1,6 @@ -import re - from sqlalchemy import Column, Float, String from .beacon import Beacon -from ogn.exceptions import OgnParseError class ReceiverBeacon(Beacon): @@ -23,67 +20,5 @@ class ReceiverBeacon(Beacon): rec_crystal_correction_fine = 0 # obsolete since 0.2.0 rec_input_noise = Column(Float) - # Pattern - version_pattern = re.compile(r"v(\d+\.\d+\.\d+)\.?(.+)?") - cpu_pattern = re.compile(r"CPU:(\d+\.\d+)") - cpu_temp_pattern = re.compile(r"([\+\-]\d+\.\d+)C") - ram_pattern = re.compile(r"RAM:(\d+\.\d+)/(\d+\.\d+)MB") - ntp_pattern = re.compile(r"NTP:(\d+\.\d+)ms/([\+\-]\d+\.\d+)ppm") - - rf_pattern_full = re.compile(r"RF:([\+\-]\d+)([\+\-]\d+\.\d+)ppm/([\+\-]\d+\.\d+)dB") - rf_pattern_light1 = re.compile(r"RF:([\+\-]\d+\.\d+)dB") - rf_pattern_light2 = re.compile(r"RF:([\+\-]\d+)([\+\-]\d+\.\d+)ppm") - - def __init__(self, beacon=None): - if beacon is not None: - self.name = beacon.name - self.receiver_name = beacon.receiver_name - self.timestamp = beacon.timestamp - self.latitude = beacon.latitude - self.longitude = beacon.longitude - self.ground_speed = beacon.ground_speed - self.track = beacon.track - self.altitude = beacon.altitude - self.comment = beacon.comment - - self.parse(beacon.comment) - - def parse(self, text): - for part in text.split(' '): - version_match = self.version_pattern.match(part) - cpu_match = self.cpu_pattern.match(part) - cpu_temp_match = self.cpu_temp_pattern.match(part) - ram_match = self.ram_pattern.match(part) - ntp_match = self.ntp_pattern.match(part) - - rf_full_match = self.rf_pattern_full.match(part) - rf_light1_match = self.rf_pattern_light1.match(part) - rf_light2_match = self.rf_pattern_light2.match(part) - - if version_match is not None: - self.version = version_match.group(1) - self.platform = version_match.group(2) - elif cpu_match is not None: - self.cpu_load = float(cpu_match.group(1)) - elif cpu_temp_match is not None: - self.cpu_temp = float(cpu_temp_match.group(1)) - elif ram_match is not None: - self.free_ram = float(ram_match.group(1)) - self.total_ram = float(ram_match.group(2)) - elif ntp_match is not None: - self.ntp_error = float(ntp_match.group(1)) - self.rt_crystal_correction = float(ntp_match.group(2)) - elif rf_full_match is not None: - self.rec_crystal_correction = int(rf_full_match.group(1)) - self.rec_crystal_correction_fine = float(rf_full_match.group(2)) - self.rec_input_noise = float(rf_full_match.group(3)) - elif rf_light1_match is not None: - self.rec_input_noise = float(rf_light1_match.group(1)) - elif rf_light2_match is not None: - self.rec_crystal_correction = int(rf_light2_match.group(1)) - self.rec_crystal_correction_fine = float(rf_light2_match.group(2)) - else: - raise OgnParseError(expected_type="ReceiverBeacon", substring=part) - def __repr__(self): return "" % (self.name, self.version) diff --git a/tests/model/__init__.py b/ogn/parser/__init__.py similarity index 100% rename from tests/model/__init__.py rename to ogn/parser/__init__.py diff --git a/ogn/exceptions.py b/ogn/parser/exceptions.py similarity index 63% rename from ogn/exceptions.py rename to ogn/parser/exceptions.py index fcbb6c1..e388e88 100644 --- a/ogn/exceptions.py +++ b/ogn/parser/exceptions.py @@ -4,26 +4,29 @@ exception definitions from datetime import datetime -class AprsParseError(Exception): +class ParseError(Exception): + pass + + +class AprsParseError(ParseError): """Parse error while parsing an aprs packet.""" def __init__(self, aprs_string): self.aprs_string = aprs_string - self.message = "This is not a valid APRS string: {}".format(aprs_string) + self.message = "This is not a valid APRS packet: {}".format(aprs_string) super(AprsParseError, self).__init__(self.message) -class OgnParseError(Exception): - """Parse error while parsing an aprs packet substring.""" - def __init__(self, substring, expected_type): - self.substring = substring - self.expected_type = expected_type +class OgnParseError(ParseError): + """Parse error while parsing an ogn message from aprs comment.""" + def __init__(self, aprs_comment): + self.aprs_comment = aprs_comment - self.message = "For type {} this is not a valid token: {}".format(expected_type, substring) + self.message = "This is not a valid OGN message: {}".format(aprs_comment) super(OgnParseError, self).__init__(self.message) -class AmbigousTimeError(Exception): +class AmbigousTimeError(ParseError): """Timstamp from the past/future, can't fully reconstruct datetime from timestamp.""" def __init__(self, reference, packet_time): self.reference = reference diff --git a/ogn/parser/parse.py b/ogn/parser/parse.py new file mode 100644 index 0000000..e0a37c6 --- /dev/null +++ b/ogn/parser/parse.py @@ -0,0 +1,84 @@ +import re +from datetime import datetime + +from ogn.parser.utils import createTimestamp, dmsToDeg, kts2kmh, feet2m, fpm2ms +from ogn.parser.pattern import PATTERN_APRS, PATTERN_RECEIVER_BEACON, PATTERN_AIRCRAFT_BEACON +from ogn.parser.exceptions import AprsParseError, OgnParseError + + +def parse_aprs(message, reference_date=None): + if reference_date is None: + reference_date = datetime.utcnow() + + match = re.search(PATTERN_APRS, message) + if match: + return {'name': match.group('callsign'), + 'receiver_name': match.group('receiver'), + 'timestamp': createTimestamp(match.group('time'), reference_date), + 'latitude': dmsToDeg(float(match.group('latitude')) / 100) * + (-1 if match.group('latitude_sign') == 'S' else 1) + + (int(match.group('latitude_enhancement')) / 1000 / 60 if match.group('latitude_enhancement') else 0), + 'symboltable': match.group('symbol_table'), + 'longitude': dmsToDeg(float(match.group('longitude')) / 100) * + (-1 if match.group('longitude_sign') == 'W' else 1) + + (int(match.group('longitude_enhancement')) / 1000 / 60 if match.group('longitude_enhancement') else 0), + 'symbolcode': match.group('symbol'), + 'track': int(match.group('course')) if match.group('course_extension') else 0, + 'ground_speed': int(match.group('ground_speed')) * kts2kmh if match.group('ground_speed') else 0, + 'altitude': int(match.group('altitude')) * feet2m, + 'comment': match.group('comment')} + + raise AprsParseError(message) + + +def parse_ogn_aircraft_beacon(aprs_comment): + ac_match = re.search(PATTERN_AIRCRAFT_BEACON, aprs_comment) + if ac_match: + return {'address_type': int(ac_match.group('details'), 16) & 0b00000011, + 'aircraft_type': (int(ac_match.group('details'), 16) & 0b01111100) >> 2, + 'stealth': (int(ac_match.group('details'), 16) & 0b10000000) >> 7 == 1, + 'address': ac_match.group('id'), + 'climb_rate': int(ac_match.group('climb_rate')) * fpm2ms, + 'turn_rate': float(ac_match.group('turn_rate')), + 'flightlevel': float(ac_match.group('flight_level')) if ac_match.group('flight_level') else None, + 'signal_strength': float(ac_match.group('signal')), + 'error_count': float(ac_match.group('errors')), + 'frequency_offset': float(ac_match.group('frequency_offset')), + 'gps_status': ac_match.group('gps_accuracy'), + 'software_version': float(ac_match.group('flarm_software_version')) if ac_match.group('flarm_software_version') else None, + 'hardware_version': int(ac_match.group('flarm_hardware_version')) if ac_match.group('flarm_hardware_version') else None, + 'real_address': ac_match.group('flarm_id')} + else: + return None + + +def parse_ogn_receiver_beacon(aprs_comment): + rec_match = re.search(PATTERN_RECEIVER_BEACON, aprs_comment) + if rec_match: + return {'version': rec_match.group('version'), + 'platform': rec_match.group('platform'), + 'cpu_load': float(rec_match.group('cpu_load')), + 'free_ram': float(rec_match.group('ram_free')), + 'total_ram': float(rec_match.group('ram_total')), + 'ntp_error': float(rec_match.group('ntp_offset')), + 'rt_crystal_correction': float(rec_match.group('ntp_correction')), + 'cpu_temp': float(rec_match.group('cpu_temperature')) if rec_match.group('cpu_temperature') else None, + 'rec_crystal_correction': int(rec_match.group('manual_correction')) if rec_match.group('manual_correction') else 0, + 'rec_crystal_correction_fine': float(rec_match.group('automatic_correction')) if rec_match.group('automatic_correction') else 0.0, + 'rec_input_noise': float(rec_match.group('input_noise')) if rec_match.group('input_noise') else None} + else: + return None + + +def parse_ogn_beacon(aprs_comment): + ac_data = parse_ogn_aircraft_beacon(aprs_comment) + if ac_data: + ac_data.update({'beacon_type': 'aircraft_beacon'}) + return ac_data + + rc_data = parse_ogn_receiver_beacon(aprs_comment) + if rc_data: + rc_data.update({'beacon_type': 'receiver_beacon'}) + return rc_data + + raise OgnParseError(aprs_comment) diff --git a/ogn/parser/pattern.py b/ogn/parser/pattern.py new file mode 100644 index 0000000..9cc04d2 --- /dev/null +++ b/ogn/parser/pattern.py @@ -0,0 +1,63 @@ +import re + + +PATTERN_APRS = re.compile(r"^(?P.+?)>APRS,.+,(?P.+?):/(?P