diff --git a/ogn/commands/database.py b/ogn/commands/database.py index c53b9c5..1e46891 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -134,9 +134,48 @@ def update_relations(): @manager.command -def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'): +def import_csv_logfile(path, logfile='main.log', loglevel='INFO'): """Import csv logfile .""" + import datetime + + import os + if os.path.isfile(path): + print("{}: Importing file: {}".format(datetime.datetime.now(), path)) + import_logfile(path) + elif os.path.isdir(path): + print("{}: Scanning path: {}".format(datetime.datetime.now(), path)) + for filename in os.listdir(path): + print("{}: Importing file: {}".format(datetime.datetime.now(), filename)) + import_logfile(os.path.join(path, filename)) + else: + print("{}: Path {} not found.".format(datetime.datetime.now(), path)) + + print("{}: Finished.".format(datetime.datetime.now())) + + +def import_logfile(path): + f = open(path, 'r') + try: + header = f.readline().strip() + except UnicodeDecodeError as e: + print("Not a text file: {}".format(path)) + f.close() + return + f.close() + + aircraft_beacon_header = ','.join(AircraftBeacon.get_csv_columns()) + receiver_beacon_header = ','.join(ReceiverBeacon.get_csv_columns()) + + if header == aircraft_beacon_header: + import_aircraft_beacon_logfile(path) + elif header == receiver_beacon_header: + import_receiver_beacon_logfile(path) + else: + print("Unknown file type: {}".format()) + + +def import_aircraft_beacon_logfile(csv_logfile): SQL_TEMPTABLE_STATEMENT = """ CREATE TABLE aircraft_beacon_temp( location geometry, @@ -178,7 +217,7 @@ def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN column_names = ','.join(AircraftBeacon.get_csv_columns()) sql = SQL_COPY_STATEMENT % column_names - print("Start importing logfile") + print("Start importing logfile: {}".format(csv_logfile)) conn = session.connection().connection cursor = conn.cursor() @@ -224,8 +263,7 @@ def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN print("Finished") -@manager.command -def import_receiver_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'): +def import_receiver_beacon_logfile(csv_logfile): """Import csv logfile .""" SQL_TEMPTABLE_STATEMENT = """ @@ -272,7 +310,7 @@ def import_receiver_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN column_names = ','.join(ReceiverBeacon.get_csv_columns()) sql = SQL_COPY_STATEMENT % column_names - print("Start importing logfile") + print("Start importing logfile: {}".format(csv_logfile)) conn = session.connection().connection cursor = conn.cursor() diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index acd7a08..50f891d 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -5,7 +5,8 @@ from ogn.gateway.process import process_beacon, message_to_beacon from datetime import datetime from manager import Manager from ogn.model import AircraftBeacon, ReceiverBeacon -import gzip + +from ogn.utils import open_file import os manager = Manager() @@ -57,7 +58,6 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'): logger = logging.getLogger(__name__) - import os if os.path.isfile(path): logger.info("Reading file: {}".format(path)) convert(path) @@ -73,20 +73,10 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'): logging.shutdown() -def opener(filename): - f = open(filename,'rb') - a = f.read(2) - f.close() - if (a == b'\x1f\x8b'): - f = gzip.open(filename, 'rt') - return f - else: - f = open(filename, 'rt') - return f - - def convert(sourcefile, path=''): import re + import csv + match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile) if match: reference_date = match.group(1) @@ -94,7 +84,7 @@ def convert(sourcefile, path=''): print("filename '{}' does not match pattern".format(sourcefile)) return - fin = opener(os.path.join(path, sourcefile)) + fin = open_file(os.path.join(path, sourcefile)) # get total lines of the input file total = 0 @@ -117,7 +107,6 @@ def convert(sourcefile, path=''): progress = -1 num_lines = 0 - import csv wr_ab = csv.writer(fout_ab, delimiter=',') wr_ab.writerow(AircraftBeacon.get_csv_columns()) diff --git a/ogn/utils.py b/ogn/utils.py index 252d23d..29d7d8e 100644 --- a/ogn/utils.py +++ b/ogn/utils.py @@ -10,6 +10,8 @@ from geopy.exc import GeopyError from aerofiles.seeyou import Reader from ogn.parser.utils import feet2m +import gzip + DDB_URL = "http://ddb.glidernet.org/download/?t=1" @@ -102,3 +104,16 @@ def get_airports(cupfile): print('Failed to parse line: {} {}'.format(line, e)) return airports + + +def open_file(filename): + """Opens a regular or unzipped textfile for reading.""" + f = open(filename,'rb') + a = f.read(2) + f.close() + if (a == b'\x1f\x8b'): + f = gzip.open(filename, 'rt') + return f + else: + f = open(filename, 'rt') + return f