diff --git a/ogn/commands/database.py b/ogn/commands/database.py index 25439d6..c53b9c5 100644 --- a/ogn/commands/database.py +++ b/ogn/commands/database.py @@ -108,6 +108,9 @@ def update_relations(): ins2 = insert(Device).from_select([Device.address], missing_addresses_query) session.execute(ins2) + session.commit() + print("Inserted {} Receivers and {} Devices".format(ins, ins2)) + return # Update AircraftBeacons upd = session.query(AircraftBeacon) \ @@ -128,3 +131,179 @@ def update_relations(): session.commit() print("Updated {} AircraftBeacons and {} ReceiverBeacons". format(upd, upd2)) + + +@manager.command +def import_aircraft_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'): + """Import csv logfile .""" + + SQL_TEMPTABLE_STATEMENT = """ + CREATE TABLE aircraft_beacon_temp( + location geometry, + altitude integer, + name character varying, + receiver_name character varying(9), + "timestamp" timestamp without time zone, + track integer, + ground_speed double precision, + + address_type smallint, + aircraft_type smallint, + stealth boolean, + address character varying(6), + climb_rate double precision, + turn_rate double precision, + flightlevel double precision, + signal_quality double precision, + error_count integer, + frequency_offset double precision, + gps_status character varying, + software_version double precision, + hardware_version smallint, + real_address character varying(6), + signal_power double precision + ) + """ + + session.execute(SQL_TEMPTABLE_STATEMENT) + + SQL_COPY_STATEMENT = """ + COPY aircraft_beacon_temp(%s) FROM STDIN WITH + CSV + HEADER + DELIMITER AS ',' + """ + + file = open(csv_logfile, 'r') + column_names = ','.join(AircraftBeacon.get_csv_columns()) + sql = SQL_COPY_STATEMENT % column_names + + print("Start importing logfile") + + conn = session.connection().connection + cursor = conn.cursor() + cursor.copy_expert(sql=sql, file=file) + conn.commit() + cursor.close() + print("Read logfile into temporary table") + + # create device if not exist + session.execute(""" + INSERT INTO device(address) + SELECT DISTINCT(t.address) + FROM aircraft_beacon_temp t + WHERE NOT EXISTS (SELECT 1 FROM device d WHERE d.address = t.address) + """) + print("Inserted missing Devices") + + # create receiver if not exist + session.execute(""" + INSERT INTO receiver(name) + SELECT DISTINCT(t.receiver_name) + FROM aircraft_beacon_temp t + WHERE NOT EXISTS (SELECT 1 FROM receiver r WHERE r.name = t.receiver_name) + """) + print("Inserted missing Receivers") + + session.execute(""" + INSERT INTO aircraft_beacon(location, altitude, name, receiver_name, timestamp, track, ground_speed, + address_type, aircraft_type, stealth, address, climb_rate, turn_rate, flightlevel, signal_quality, error_count, frequency_offset, gps_status, software_version, hardware_version, real_address, signal_power, + status, receiver_id, device_id) + SELECT t.location, t.altitude, t.name, t.receiver_name, t.timestamp, t.track, t.ground_speed, + t.address_type, t.aircraft_type, t.stealth, t.address, t.climb_rate, t.turn_rate, t.flightlevel, t.signal_quality, t.error_count, t.frequency_offset, t.gps_status, t.software_version, t.hardware_version, t.real_address, t.signal_power, + 0, r.id, d.id + FROM aircraft_beacon_temp t, receiver r, device d + WHERE t.receiver_name = r.name AND t.address = d.address + """) + print("Wrote AircraftBeacons from temporary table into final table") + + session.execute("""DROP TABLE aircraft_beacon_temp""") + print("Dropped temporary table") + + session.commit() + print("Finished") + + +@manager.command +def import_receiver_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'): + """Import csv logfile .""" + + SQL_TEMPTABLE_STATEMENT = """ + CREATE TABLE receiver_beacon_temp( + location geometry, + altitude integer, + name character varying, + receiver_name character varying(9), + "timestamp" timestamp without time zone, + track integer, + ground_speed double precision, + + version character varying, + platform character varying, + cpu_load double precision, + free_ram double precision, + total_ram double precision, + ntp_error double precision, + rt_crystal_correction double precision, + voltage double precision, + amperage double precision, + cpu_temp double precision, + senders_visible integer, + senders_total integer, + rec_input_noise double precision, + senders_signal double precision, + senders_messages integer, + good_senders_signal double precision, + good_senders integer, + good_and_bad_senders integer + ) + """ + + session.execute(SQL_TEMPTABLE_STATEMENT) + + SQL_COPY_STATEMENT = """ + COPY receiver_beacon_temp(%s) FROM STDIN WITH + CSV + HEADER + DELIMITER AS ',' + """ + + file = open(csv_logfile, 'r') + column_names = ','.join(ReceiverBeacon.get_csv_columns()) + sql = SQL_COPY_STATEMENT % column_names + + print("Start importing logfile") + + conn = session.connection().connection + cursor = conn.cursor() + cursor.copy_expert(sql=sql, file=file) + conn.commit() + cursor.close() + print("Read logfile into temporary table") + + # create receiver if not exist + session.execute(""" + INSERT INTO receiver(name) + SELECT DISTINCT(t.name) + FROM receiver_beacon_temp t + WHERE NOT EXISTS (SELECT 1 FROM receiver r WHERE r.name = t.name) + """) + print("Inserted missing Receivers") + + session.execute(""" + INSERT INTO receiver_beacon(location, altitude, name, receiver_name, timestamp, track, ground_speed, + version, platform, cpu_load, free_ram, total_ram, ntp_error, rt_crystal_correction, voltage,amperage, cpu_temp, senders_visible, senders_total, rec_input_noise, senders_signal, senders_messages, good_senders_signal, good_senders, good_and_bad_senders, + status, receiver_id) + SELECT t.location, t.altitude, t.name, t.receiver_name, t.timestamp, t.track, t.ground_speed, + t.version, t.platform, t.cpu_load, t.free_ram, t.total_ram, t.ntp_error, t.rt_crystal_correction, t.voltage,amperage, t.cpu_temp, t.senders_visible, t.senders_total, t.rec_input_noise, t.senders_signal, t.senders_messages, t.good_senders_signal, t.good_senders, t.good_and_bad_senders, + 0, r.id + FROM receiver_beacon_temp t, receiver r + WHERE t.name = r.name + """) + print("Wrote ReceiverBeacons from temporary table into final table") + + session.execute("""DROP TABLE receiver_beacon_temp""") + print("Dropped temporary table") + + session.commit() + print("Finished") diff --git a/ogn/gateway/manage.py b/ogn/gateway/manage.py index dfaf24a..25ff0b9 100644 --- a/ogn/gateway/manage.py +++ b/ogn/gateway/manage.py @@ -2,7 +2,6 @@ import logging from ogn.client import AprsClient from ogn.gateway.process import process_beacon, message_to_beacon -from ogn.commands.dbutils import session from datetime import datetime from manager import Manager from ogn.model import AircraftBeacon, ReceiverBeacon @@ -44,47 +43,6 @@ def run(aprs_user='anon-dev', logfile='main.log', loglevel='INFO'): logging.shutdown() -@manager.command -def import_beacon_logfile(csv_logfile, logfile='main.log', loglevel='INFO'): - """Import csv logfile .""" - - # Enable logging - log_handlers = [logging.StreamHandler()] - if logfile: - log_handlers.append(logging.FileHandler(logfile)) - logging.basicConfig(format=logging_formatstr, level=loglevel, handlers=log_handlers) - - logger = logging.getLogger(__name__) - - SQL_STATEMENT = """ - COPY %s(%s) FROM STDIN WITH - CSV - HEADER - DELIMITER AS ',' - """ - - file = open(csv_logfile, 'r') - first_line = file.readline().strip() - aircraft_beacon_columns = ','.join(AircraftBeacon.get_csv_columns()) - receiver_beacon_columns = ','.join(ReceiverBeacon.get_csv_columns()) - if first_line == aircraft_beacon_columns: - sql = SQL_STATEMENT % ('aircraft_beacon', aircraft_beacon_columns) - elif first_line == receiver_beacon_columns: - sql = SQL_STATEMENT % ('receiver_beacon', receiver_beacon_columns) - else: - print("Not a valid logfile: {}".format(csv_logfile)) - return - - logger.info("Reading logfile") - - conn = session.connection().connection - cursor = conn.cursor() - cursor.copy_expert(sql=sql, file=file) - conn.commit() - cursor.close() - logger.info("Import finished") - - # get total lines of the input file def file_len(fname): with open(fname) as f: @@ -130,7 +88,6 @@ def convert_logfile(ogn_logfile, logfile='main.log', loglevel='INFO'): progress = -1 num_lines = 0 - from ogn.model import AircraftBeacon, ReceiverBeacon import csv wr_ab = csv.writer(fout_ab, delimiter=',') wr_ab.writerow(AircraftBeacon.get_csv_columns()) diff --git a/ogn/model/aircraft_beacon.py b/ogn/model/aircraft_beacon.py index 909b33c..aa9f6d3 100644 --- a/ogn/model/aircraft_beacon.py +++ b/ogn/model/aircraft_beacon.py @@ -24,7 +24,7 @@ class AircraftBeacon(Beacon): real_address = Column(String(6)) signal_power = Column(Float) - status = Column(SmallInteger) + status = Column(SmallInteger, index=True) # Relations receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True) diff --git a/ogn/model/receiver_beacon.py b/ogn/model/receiver_beacon.py index d4a1dc3..e008467 100644 --- a/ogn/model/receiver_beacon.py +++ b/ogn/model/receiver_beacon.py @@ -29,7 +29,7 @@ class ReceiverBeacon(Beacon): good_senders = Column(Integer) good_and_bad_senders = Column(Integer) - status = Column(SmallInteger) + status = Column(SmallInteger, index=True) # Relations receiver_id = Column(Integer, ForeignKey('receiver.id', ondelete='SET NULL'), index=True)