Import logfile parameter can be a folder (with gzipped or unzipped files) or a file (gzipped or unzipped)

ogn/commands/database.py

@@ -1,6 +1,6 @@
 from ogn.commands.dbutils import engine, session
 from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
-from ogn.utils import get_airports
+from ogn.utils import get_airports, open_file
 from ogn.collect.database import update_device_infos
 
 from sqlalchemy import insert, distinct
@@ -155,13 +155,8 @@ def import_csv_logfile(path, logfile='main.log', loglevel='INFO'):
 
 
 def import_logfile(path):
-    f = open(path, 'r')
-    try:
-        header = f.readline().strip()
-    except UnicodeDecodeError as e:
-        print("Not a text file: {}".format(path))
-        f.close()
-        return
+    f = open_file(path)
+    header = f.readline().strip()
     f.close()
 
     aircraft_beacon_header = ','.join(AircraftBeacon.get_csv_columns())
@@ -214,7 +209,7 @@ def import_aircraft_beacon_logfile(csv_logfile):
         DELIMITER AS ','
     """
 
-    file = open(csv_logfile, 'r')
+    file = open_file(csv_logfile)
     column_names = ','.join(AircraftBeacon.get_csv_columns())
     sql = SQL_COPY_STATEMENT % column_names
 
@@ -225,6 +220,7 @@ def import_aircraft_beacon_logfile(csv_logfile):
     cursor.copy_expert(sql=sql, file=file)
     conn.commit()
     cursor.close()
+    file.close()
     print("Read logfile into temporary table")
 
     # create device if not exist
@@ -308,7 +304,7 @@ def import_receiver_beacon_logfile(csv_logfile):
         DELIMITER AS ','
     """
 
-    file = open(csv_logfile, 'r')
+    file = open_file(csv_logfile)
     column_names = ','.join(ReceiverBeacon.get_csv_columns())
     sql = SQL_COPY_STATEMENT % column_names
 
@@ -319,6 +315,7 @@ def import_receiver_beacon_logfile(csv_logfile):
     cursor.copy_expert(sql=sql, file=file)
     conn.commit()
     cursor.close()
+    file.close()
     print("Read logfile into temporary table")
 
     # create receiver if not exist

ogn/gateway/manage.py

@@ -76,12 +76,14 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'):
 def convert(sourcefile, path=''):
     import re
     import csv
+    import gzip
 
     match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile)
     if match:
-        reference_date = match.group(1)
+        reference_date_string = match.group(1)
+        reference_date = datetime.strptime(reference_date_string, "%Y-%m-%d")
     else:
-        print("filename '{}' does not match pattern".format(sourcefile))
+        print("filename '{}' does not match pattern. Skipping".format(sourcefile))
         return
 
     fin = open_file(os.path.join(path, sourcefile))
@@ -92,13 +94,14 @@ def convert(sourcefile, path=''):
         total += 1
     fin.seek(0)
 
-    fout_ab = open(os.path.join(path, 'aircraft_beacons.csv_' + reference_date), 'w')
-    fout_rb = open(os.path.join(path, 'receiver_beacons.csv_' + reference_date), 'w')
+    aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
+    receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
 
-    try:
-        reference_date = datetime.strptime(reference_date, "%Y-%m-%d")
-    except:
-        print('\nError in reference_date argument', reference_date)
+    if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
+        fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
+        fout_rb = gzip.open(receiver_beacon_filename, 'wt')
+    else:
+        print("Output files already exists. Skipping")
         return
 
     aircraft_beacons = list()