Move convert_logfile to bulkimport module

pull/68/head
Konstantin Gründger 2017-06-23 08:13:51 +02:00
rodzic 82e92bc1e0
commit 5a4ce57db9
2 zmienionych plików z 99 dodań i 104 usunięć

Wyświetl plik

@ -2,14 +2,113 @@ from ogn.commands.dbutils import engine, session
from ogn.model import Base, AddressOrigin, AircraftBeacon, ReceiverBeacon, Device, Receiver
from ogn.utils import get_airports, open_file
from ogn.collect.database import update_device_infos
from ogn.gateway.process import message_to_beacon
from sqlalchemy import insert, distinct
from sqlalchemy.sql import null
import os
from manager import Manager
manager = Manager()
@manager.command
def convert_logfile(path, logfile='main.log', loglevel='INFO'):
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
if os.path.isfile(path):
print("Reading file: {}".format(path))
convert(path)
print("Finished")
elif os.path.isdir(path):
for filename in os.listdir(path):
print("Reading file: {}".format(filename))
convert(filename, path=path)
print("Finished")
else:
print("Not a file nor a path: {}".format(path))
def convert(sourcefile, path=''):
import re
import csv
import gzip
import datetime
match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile)
if match:
reference_date_string = match.group(1)
reference_date = datetime.strptime(reference_date_string, "%Y-%m-%d")
else:
print("filename '{}' does not match pattern. Skipping".format(sourcefile))
return
fin = open_file(os.path.join(path, sourcefile))
# get total lines of the input file
total = 0
for line in fin:
total += 1
fin.seek(0)
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
else:
print("Output files already exists. Skipping")
return
aircraft_beacons = list()
receiver_beacons = list()
progress = -1
num_lines = 0
wr_ab = csv.writer(fout_ab, delimiter=',')
wr_ab.writerow(AircraftBeacon.get_csv_columns())
wr_rb = csv.writer(fout_rb, delimiter=',')
wr_rb.writerow(ReceiverBeacon.get_csv_columns())
print('Start importing ogn-logfile')
for line in fin:
num_lines += 1
if int(100 * num_lines / total) != progress:
progress = round(100 * num_lines / total)
print("\rReading line {} ({}%)".format(num_lines, progress), end='')
if len(aircraft_beacons) > 0:
for beacon in aircraft_beacons:
wr_ab.writerow(beacon.get_csv_values())
aircraft_beacons = list()
if len(receiver_beacons) > 0:
for beacon in receiver_beacons:
wr_rb.writerow(beacon.get_csv_values())
receiver_beacons = list()
beacon = message_to_beacon(line.strip(), reference_date=reference_date)
if beacon is not None:
if isinstance(beacon, AircraftBeacon):
aircraft_beacons.append(beacon)
elif isinstance(beacon, ReceiverBeacon):
receiver_beacons.append(beacon)
if len(aircraft_beacons) > 0:
for beacon in aircraft_beacons:
wr_ab.writerow(beacon.get_csv_values())
if len(receiver_beacons) > 0:
for beacon in receiver_beacons:
wr_rb.writerow(beacon.get_csv_values())
fin.close()
fout_ab.close()
fout_rb.close()
@manager.command
def drop_indices():
"""Drop indices of AircraftBeacon."""

Wyświetl plik

@ -44,107 +44,3 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'):
client.disconnect()
logging.shutdown()
@manager.command
def convert_logfile(path, logfile='main.log', loglevel='INFO'):
"""Convert ogn logfiles to csv logfiles (one for aircraft beacons and one for receiver beacons) <arg: path>. Logfile name: blablabla.txt_YYYY-MM-DD."""
# Enable logging
log_handlers = [logging.StreamHandler()]
if logfile:
log_handlers.append(logging.FileHandler(logfile))
logging.basicConfig(format=logging_formatstr, level=loglevel, handlers=log_handlers)
logger = logging.getLogger(__name__)
if os.path.isfile(path):
logger.info("Reading file: {}".format(path))
convert(path)
logger.info("Finished")
elif os.path.isdir(path):
for filename in os.listdir(path):
logger.info("Reading file: {}".format(filename))
convert(filename, path=path)
logger.info("Finished")
else:
print("Not a file nor a path: {}".format(path))
logging.shutdown()
def convert(sourcefile, path=''):
import re
import csv
import gzip
match = re.search('^.+\.txt\_(\d{4}\-\d{2}\-\d{2})(\.gz)?$', sourcefile)
if match:
reference_date_string = match.group(1)
reference_date = datetime.strptime(reference_date_string, "%Y-%m-%d")
else:
print("filename '{}' does not match pattern. Skipping".format(sourcefile))
return
fin = open_file(os.path.join(path, sourcefile))
# get total lines of the input file
total = 0
for line in fin:
total += 1
fin.seek(0)
aircraft_beacon_filename = os.path.join(path, 'aircraft_beacons.csv_' + reference_date_string + '.gz')
receiver_beacon_filename = os.path.join(path, 'receiver_beacons.csv_' + reference_date_string + '.gz')
if not os.path.exists(aircraft_beacon_filename) and not os.path.exists(receiver_beacon_filename):
fout_ab = gzip.open(aircraft_beacon_filename, 'wt')
fout_rb = gzip.open(receiver_beacon_filename, 'wt')
else:
print("Output files already exists. Skipping")
return
aircraft_beacons = list()
receiver_beacons = list()
progress = -1
num_lines = 0
wr_ab = csv.writer(fout_ab, delimiter=',')
wr_ab.writerow(AircraftBeacon.get_csv_columns())
wr_rb = csv.writer(fout_rb, delimiter=',')
wr_rb.writerow(ReceiverBeacon.get_csv_columns())
print('Start importing ogn-logfile')
for line in fin:
num_lines += 1
if int(100 * num_lines / total) != progress:
progress = round(100 * num_lines / total)
print("\rReading line {} ({}%)".format(num_lines, progress), end='')
if len(aircraft_beacons) > 0:
for beacon in aircraft_beacons:
wr_ab.writerow(beacon.get_csv_values())
aircraft_beacons = list()
if len(receiver_beacons) > 0:
for beacon in receiver_beacons:
wr_rb.writerow(beacon.get_csv_values())
receiver_beacons = list()
beacon = message_to_beacon(line.strip(), reference_date=reference_date)
if beacon is not None:
if isinstance(beacon, AircraftBeacon):
aircraft_beacons.append(beacon)
elif isinstance(beacon, ReceiverBeacon):
receiver_beacons.append(beacon)
if len(aircraft_beacons) > 0:
for beacon in aircraft_beacons:
wr_ab.writerow(beacon.get_csv_values())
if len(receiver_beacons) > 0:
for beacon in receiver_beacons:
wr_rb.writerow(beacon.get_csv_values())
fin.close()
fout_ab.close()
fout_rb.close()