diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 29bb4c1..372deff 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -1,6 +1,6 @@ from ogn.commands.dbutils import engine, session from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver -from ogn.utils import get_airports +from ogn.utils import get_airports, open_file from ogn.collect.database import update_device_infos from sqlalchemy import insert, distinct @@ -155,13 +155,8 @@ def import_csv_logfile(path, logfile='main.log', loglevel='INFO'): def import_logfile(path): - f = open(path, 'r') - try: - header = f.readline().strip() - except UnicodeDecodeError as e: - print("Not a text file: {}".format(path)) - f.close() - return + f = open_file(path) + header = f.readline().strip() f.close() aircraft_beacon_header = ','.join(AircraftBeacon.get_csv_columns()) @@ -214,7 +209,7 @@ def import_aircraft_beacon_logfile(csv_logfile): DELIMITER AS ',' """ - file = open(csv_logfile, 'r') + file = open_file(csv_logfile) column_names = ','.join(AircraftBeacon.get_csv_columns()) sql = SQL_COPY_STATEMENT % column_names @@ -225,6 +220,7 @@ def import_aircraft_beacon_logfile(csv_logfile): cursor.copy_expert(sql=sql, file=file) conn.commit() cursor.close() + file.close() print("Read logfile into temporary table") # create device if not exist @@ -308,7 +304,7 @@ def import_receiver_beacon_logfile(csv_logfile): DELIMITER AS ',' """ - file = open(csv_logfile, 'r') + file = open_file(csv_logfile) column_names = ','.join(ReceiverBeacon.get_csv_columns()) sql = SQL_COPY_STATEMENT % column_names @@ -319,6 +315,7 @@ def import_receiver_beacon_logfile(csv_logfile): cursor.copy_expert(sql=sql, file=file) conn.commit() cursor.close() + file.close() print("Read logfile into temporary table") # create receiver if not exist diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index ba52afc..2d6bcfc 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -76,12 +76,14 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'): def convert(sourcefile, path=''): import re import csv + import gzip match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile) if match: - reference_date = match.group(1) + reference_date_string = match.group(1) + reference_date = datetime.strptime(reference_date_string, "%Y-%m-%d") else: - print("filename '{}' does not match pattern".format(sourcefile)) + print("filename '{}' does not match pattern. Skipping".format(sourcefile)) return fin = open_file(os.path.join(path, sourcefile)) @@ -92,13 +94,14 @@ def convert(sourcefile, path=''): total += 1 fin.seek(0) - fout_ab = open(os.path.join(path, 'aircraft_beacons.csv_' + reference_date), 'w') - fout_rb = open(os.path.join(path, 'receiver_beacons.csv_' + reference_date), 'w') + aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz') + receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz') - try: - reference_date = datetime.strptime(reference_date, "%Y-%m-%d") - except: - print('\nError in reference_date argument', reference_date) + if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename): + fout_ab = gzip.open(aircraft_beacon_filename, 'wt') + fout_rb = gzip.open(receiver_beacon_filename, 'wt') + else: + print("Output files already exists. Skipping") return aircraft_beacons = list()