Move opener to utils and implement batch cdv import

ogn/commands/database.py

@@ -134,9 +134,48 @@ def update_relations():
 
 
 @manager.command
-def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'):
+def import_csv_logfile(path, logfile='main.log', loglevel='INFO'):
     """Import csv logfile <arg: csv logfile>."""
 
+    import datetime
+
+    import os
+    if os.path.isfile(path):
+        print("{}: Importing file: {}".format(datetime.datetime.now(), path))
+        import_logfile(path)
+    elif os.path.isdir(path):
+        print("{}: Scanning path: {}".format(datetime.datetime.now(), path))
+        for filename in os.listdir(path):
+            print("{}: Importing file: {}".format(datetime.datetime.now(), filename))
+            import_logfile(os.path.join(path, filename))
+    else:
+        print("{}: Path {} not found.".format(datetime.datetime.now(), path))
+
+    print("{}: Finished.".format(datetime.datetime.now()))
+
+
+def import_logfile(path):
+    f = open(path, 'r')
+    try:
+        header = f.readline().strip()
+    except UnicodeDecodeError as e:
+        print("Not a text file: {}".format(path))
+        f.close()
+        return
+    f.close()
+
+    aircraft_beacon_header = ','.join(AircraftBeacon.get_csv_columns())
+    receiver_beacon_header = ','.join(ReceiverBeacon.get_csv_columns())
+
+    if header == aircraft_beacon_header:
+        import_aircraft_beacon_logfile(path)
+    elif header == receiver_beacon_header:
+        import_receiver_beacon_logfile(path)
+    else:
+        print("Unknown file type: {}".format())
+
+
+def import_aircraft_beacon_logfile(csv_logfile):
     SQL_TEMPTABLE_STATEMENT = """
     CREATE TABLE aircraft_beacon_temp(
         location geometry,
@@ -178,7 +217,7 @@ def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN
     column_names = ','.join(AircraftBeacon.get_csv_columns())
     sql = SQL_COPY_STATEMENT % column_names
 
-    print("Start importing logfile")
+    print("Start importing logfile: {}".format(csv_logfile))
 
     conn = session.connection().connection
     cursor = conn.cursor()
@@ -224,8 +263,7 @@ def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN
     print("Finished")
 
 
-@manager.command
-def import_receiver_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'):
+def import_receiver_beacon_logfile(csv_logfile):
     """Import csv logfile <arg: csv logfile>."""
 
     SQL_TEMPTABLE_STATEMENT = """
@@ -272,7 +310,7 @@ def import_receiver_beacon_logfile(csv_logfile, logfile='main.log', loglevel='IN
     column_names = ','.join(ReceiverBeacon.get_csv_columns())
     sql = SQL_COPY_STATEMENT % column_names
 
-    print("Start importing logfile")
+    print("Start importing logfile: {}".format(csv_logfile))
 
     conn = session.connection().connection
     cursor = conn.cursor()

ogn/gateway/manage.py

@@ -5,7 +5,8 @@ from ogn.gateway.process import process_beacon, message_to_beacon
 from datetime import datetime
 from manager import Manager
 from ogn.model import AircraftBeacon, ReceiverBeacon
-import gzip
+
+from ogn.utils import open_file
 import os
 
 manager = Manager()
@@ -57,7 +58,6 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'):
 
     logger = logging.getLogger(__name__)
 
-    import os
     if os.path.isfile(path):
         logger.info("Reading file: {}".format(path))
         convert(path)
@@ -73,20 +73,10 @@ def convert_logfile(path, logfile='main.log', loglevel='INFO'):
     logging.shutdown()
 
 
-def opener(filename):
-    f = open(filename,'rb')
-    a = f.read(2)
-    f.close()
-    if (a == b'\x1f\x8b'):
-        f = gzip.open(filename, 'rt')
-        return f
-    else:
-        f = open(filename, 'rt')
-        return f
-
-
 def convert(sourcefile, path=''):
     import re
+    import csv
+
     match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile)
     if match:
         reference_date = match.group(1)
@@ -94,7 +84,7 @@ def convert(sourcefile, path=''):
         print("filename '{}' does not match pattern".format(sourcefile))
         return
 
-    fin = opener(os.path.join(path, sourcefile))
+    fin = open_file(os.path.join(path, sourcefile))
 
     # get total lines of the input file
     total = 0
@@ -117,7 +107,6 @@ def convert(sourcefile, path=''):
     progress = -1
     num_lines = 0
 
-    import csv
     wr_ab = csv.writer(fout_ab, delimiter=',')
     wr_ab.writerow(AircraftBeacon.get_csv_columns())
 

ogn/utils.py

@@ -10,6 +10,8 @@ from geopy.exc import GeopyError
 from aerofiles.seeyou import Reader
 from ogn.parser.utils import feet2m
 
+import gzip
+
 DDB_URL = "http://ddb.glidernet.org/download/?t=1"
 
 
@@ -102,3 +104,16 @@ def get_airports(cupfile):
                 print('Failed to parse line: {} {}'.format(line, e))
 
     return airports
+
+
+def open_file(filename):
+    """Opens a regular or unzipped textfile for reading."""
+    f = open(filename,'rb')
+    a = f.read(2)
+    f.close()
+    if (a == b'\x1f\x8b'):
+        f = gzip.open(filename, 'rt')
+        return f
+    else:
+        f = open(filename, 'rt')
+        return f