From fe83440eb4b1a5145aa64639f3603fc37b016ff9 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 20:56:59 +0100 Subject: [PATCH 1/6] Add missing dependency on alembic. --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 9de28bf..b0ee5fd 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ $ apt-get install redis-server 3. Create database ``` $ ./manage.py db.init +$ alembic stamp head ``` ## Running the aprs client and task server diff --git a/requirements.txt b/requirements.txt index 2e74710..2ed53be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ coveralls==0.4.4 geopy==1.11.0 manage.py==0.2.10 celery[redis]>=3.1,<3.2 +alembic==0.8.3 From b391fca45bd450f726f0e6daf0c7f58febedcb36 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 21:23:25 +0100 Subject: [PATCH 2/6] Fix show.receiver statistics. Use current time for calculations, not the start-time of celery. --- ogn/commands/showreceiver.py | 37 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/ogn/commands/showreceiver.py b/ogn/commands/showreceiver.py index e5f088a..9f210f4 100644 --- a/ogn/commands/showreceiver.py +++ b/ogn/commands/showreceiver.py @@ -1,26 +1,29 @@ from datetime import datetime, timedelta - from sqlalchemy.sql import func from sqlalchemy import distinct, and_ from ogn.model import ReceiverBeacon, Receiver - from ogn.commands.dbutils import session -back_24h = datetime.utcnow() - timedelta(days=1) -receiver_messages_per_24h = 24 * 60 / 5 - from manager import Manager manager = Manager() +receiver_beacons_per_day = 24 * 60 / 5 + @manager.command def list_all(): """Show a list of all receivers.""" - sq = session.query(distinct(ReceiverBeacon.name).label('name'), func.max(ReceiverBeacon.timestamp).label('lastseen'), func.count(ReceiverBeacon.name).label('messages_count')).\ - filter(ReceiverBeacon.timestamp > back_24h).\ - group_by(ReceiverBeacon.name).\ - subquery() + + timestamp_24h_ago = datetime.utcnow() - timedelta(days=1) + + sq = session.query( + distinct(ReceiverBeacon.name).label('name'), + func.max(ReceiverBeacon.timestamp).label('lastseen'), + func.count(ReceiverBeacon.name).label('messages_count')).\ + filter(ReceiverBeacon.timestamp > timestamp_24h_ago).\ + group_by(ReceiverBeacon.name).\ + subquery() query = session.query(Receiver, sq.c.messages_count).\ filter(Receiver.name == sq.c.name).\ @@ -28,14 +31,21 @@ def list_all(): print('--- Receivers ---') for [receiver, messages_count] in query.all(): - print('%9s (%2s): %3d%% avail, %s, %s ' % (receiver.name, receiver.country_code, 100.0*float(messages_count/receiver_messages_per_24h), receiver.version, receiver.platform)) + print('%9s (%2s): %3d%% avail, %s, %s ' % (receiver.name, + receiver.country_code, + 100.0 * float(messages_count / receiver_beacons_per_day), + receiver.version, + receiver.platform)) @manager.command def software_stats(): """Show some statistics of receiver software.""" + + timestamp_24h_ago = datetime.utcnow() - timedelta(days=1) + sq = session.query(ReceiverBeacon.name, func.max(ReceiverBeacon.timestamp).label('lastseen')).\ - filter(ReceiverBeacon.timestamp > back_24h).\ + filter(ReceiverBeacon.timestamp > timestamp_24h_ago).\ group_by(ReceiverBeacon.name).\ subquery() @@ -52,8 +62,11 @@ def software_stats(): @manager.command def hardware_stats(): """Show some statistics of receiver hardware.""" + + timestamp_24h_ago = datetime.utcnow() - timedelta(days=1) + sq = session.query(ReceiverBeacon.name, func.max(ReceiverBeacon.timestamp).label('lastseen')).\ - filter(ReceiverBeacon.timestamp > back_24h).\ + filter(ReceiverBeacon.timestamp > timestamp_24h_ago).\ group_by(ReceiverBeacon.name).\ subquery() From 316f71c8e4909f2266d2c3023ad69b13b244074b Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 21:37:41 +0100 Subject: [PATCH 3/6] Fix pep8. --- ogn/commands/showreceiver.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ogn/commands/showreceiver.py b/ogn/commands/showreceiver.py index 9f210f4..1cdb43e 100644 --- a/ogn/commands/showreceiver.py +++ b/ogn/commands/showreceiver.py @@ -17,13 +17,10 @@ def list_all(): timestamp_24h_ago = datetime.utcnow() - timedelta(days=1) - sq = session.query( - distinct(ReceiverBeacon.name).label('name'), - func.max(ReceiverBeacon.timestamp).label('lastseen'), - func.count(ReceiverBeacon.name).label('messages_count')).\ - filter(ReceiverBeacon.timestamp > timestamp_24h_ago).\ - group_by(ReceiverBeacon.name).\ - subquery() + sq = session.query(distinct(ReceiverBeacon.name).label('name'), + func.max(ReceiverBeacon.timestamp).label('lastseen'), + func.count(ReceiverBeacon.name).label('messages_count') + ).filter(ReceiverBeacon.timestamp > timestamp_24h_ago).group_by(ReceiverBeacon.name).subquery() query = session.query(Receiver, sq.c.messages_count).\ filter(Receiver.name == sq.c.name).\ From ec80b9086344b29d6dd7be7e1cd2a9dc894cd240 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 21:39:13 +0100 Subject: [PATCH 4/6] travis ci: Add pep8 check. --- .travis.yml | 3 +++ requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc2fd86..94f9b6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ language: python python: - 3.4 +before_script: + - "pep8 --ignore=E501 --exclude=logbook.py,utils.py ogn" + script: - nosetests --with-coverage --cover-package=ogn diff --git a/requirements.txt b/requirements.txt index 2ed53be..72a3314 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -SQLAlchemy==1.0.8 nose==1.3.7 coveralls==0.4.4 +pep8==1.6.2 +SQLAlchemy==1.0.8 geopy==1.11.0 manage.py==0.2.10 celery[redis]>=3.1,<3.2 From 57f51ec111ac579ca855a09aee8be052b32d1271 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 21:51:23 +0100 Subject: [PATCH 5/6] Fix pep8 again. --- ogn/commands/database.py | 5 ++--- ogn/gateway/manage.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 0185f30..2935b9c 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -1,15 +1,14 @@ from ogn.model import Base +from ogn.collect.fetchddb import update_ddb_from_ogn, update_ddb_from_file +from ogn.commands.dbutils import engine from manager import Manager manager = Manager() -from ogn.collect.fetchddb import update_ddb_from_ogn, update_ddb_from_file - @manager.command def init(): """Initialize the database.""" - from ogn.commands.dbutils import engine Base.metadata.create_all(engine) print("Done.") diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index ce50a83..b813775 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -3,15 +3,16 @@ import socket from ogn.gateway import ognGateway from ogn.logger import logger -DB_URI = 'sqlite:///beacons.db' - from manager import Manager manager = Manager() +DB_URI = 'sqlite:///beacons.db' + @manager.command def run(aprs_user="anon-dev"): """Run the aprs client.""" + if len(aprs_user) < 3 or len(aprs_user) > 9: print("aprs_user must be a string of 3-9 characters") return From 6af31fe7a71da3f9981c106c8dd54d354d18fc03 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sun, 29 Nov 2015 21:59:34 +0100 Subject: [PATCH 6/6] Move ogn client (gateway) to a separate file. The documentation[1] says: > In the simplest case, __init__.py can just be an empty file, > but it can also execute initialization code for the package or > set the __all__ variable, described later. A class like 'ognGateway' can't be considered 'initialization code'. [1]: https://docs.python.org/3/tutorial/modules.html#packages --- ogn/gateway/__init__.py | 69 ----------------------------------------- ogn/gateway/client.py | 69 +++++++++++++++++++++++++++++++++++++++++ ogn/gateway/manage.py | 2 +- 3 files changed, 70 insertions(+), 70 deletions(-) create mode 100644 ogn/gateway/client.py diff --git a/ogn/gateway/__init__.py b/ogn/gateway/__init__.py index 5a6e4ec..e69de29 100644 --- a/ogn/gateway/__init__.py +++ b/ogn/gateway/__init__.py @@ -1,69 +0,0 @@ -import socket -from time import time - -from ogn.gateway import settings -from ogn.commands.dbutils import session -from ogn.aprs_parser import parse_aprs -from ogn.aprs_utils import create_aprs_login -from ogn.exceptions import AprsParseError, OgnParseError -from ogn.logger import logger - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from ogn.model import Base - - -class ognGateway: - def __init__(self): - pass - - def connect_db(self): - self.session = session - - def connect(self, aprs_user): - # create socket, connect to server, login and make a file object associated with the socket - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - self.sock.connect((settings.APRS_SERVER_HOST, settings.APRS_SERVER_PORT)) - - login = create_aprs_login(aprs_user, -1, settings.APRS_APP_NAME, settings.APRS_APP_VER, settings.APRS_FILTER) - self.sock.send(login.encode()) - self.sock_file = self.sock.makefile('rw') - - def disconnect(self): - # close everything - self.sock.shutdown(0) - self.sock.close() - - def run(self): - keepalive_time = time() - while True: - if time() - keepalive_time > settings.APRS_KEEPALIVE_TIME: - logger.debug('Sending keepalive') - self.sock.send("#keepalive".encode()) - keepalive_time = time() - - # Read packet string from socket - packet_str = self.sock_file.readline().strip() - - # A zero length line should not be return if keepalives are being sent - # A zero length line will only be returned after ~30m if keepalives are not sent - if len(packet_str) == 0: - logger.warning('Read returns zero length string. Failure. Orderly closeout') - break - - self.proceed_line(packet_str) - - def proceed_line(self, line): - try: - beacon = parse_aprs(line) - except AprsParseError: - logger.error('AprsParseError while parsing line: %s' % line, exc_info=True) - return - except OgnParseError: - logger.error('OgnParseError while parsing line: ' % line, exc_info=True) - return - - if beacon is not None: - self.session.add(beacon) - self.session.commit() diff --git a/ogn/gateway/client.py b/ogn/gateway/client.py new file mode 100644 index 0000000..5a6e4ec --- /dev/null +++ b/ogn/gateway/client.py @@ -0,0 +1,69 @@ +import socket +from time import time + +from ogn.gateway import settings +from ogn.commands.dbutils import session +from ogn.aprs_parser import parse_aprs +from ogn.aprs_utils import create_aprs_login +from ogn.exceptions import AprsParseError, OgnParseError +from ogn.logger import logger + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from ogn.model import Base + + +class ognGateway: + def __init__(self): + pass + + def connect_db(self): + self.session = session + + def connect(self, aprs_user): + # create socket, connect to server, login and make a file object associated with the socket + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self.sock.connect((settings.APRS_SERVER_HOST, settings.APRS_SERVER_PORT)) + + login = create_aprs_login(aprs_user, -1, settings.APRS_APP_NAME, settings.APRS_APP_VER, settings.APRS_FILTER) + self.sock.send(login.encode()) + self.sock_file = self.sock.makefile('rw') + + def disconnect(self): + # close everything + self.sock.shutdown(0) + self.sock.close() + + def run(self): + keepalive_time = time() + while True: + if time() - keepalive_time > settings.APRS_KEEPALIVE_TIME: + logger.debug('Sending keepalive') + self.sock.send("#keepalive".encode()) + keepalive_time = time() + + # Read packet string from socket + packet_str = self.sock_file.readline().strip() + + # A zero length line should not be return if keepalives are being sent + # A zero length line will only be returned after ~30m if keepalives are not sent + if len(packet_str) == 0: + logger.warning('Read returns zero length string. Failure. Orderly closeout') + break + + self.proceed_line(packet_str) + + def proceed_line(self, line): + try: + beacon = parse_aprs(line) + except AprsParseError: + logger.error('AprsParseError while parsing line: %s' % line, exc_info=True) + return + except OgnParseError: + logger.error('OgnParseError while parsing line: ' % line, exc_info=True) + return + + if beacon is not None: + self.session.add(beacon) + self.session.commit() diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index b813775..e243811 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -1,6 +1,6 @@ import socket -from ogn.gateway import ognGateway +from ogn.gateway.client import ognGateway from ogn.logger import logger from manager import Manager