ogn-python/app/gateway/process_tools.py

import os
import gzip
import time
from contextlib import contextmanager

from flask import current_app
from app import db


@contextmanager
def open_file(filename):
    """Opens a regular OR gzipped textfile for reading."""

    file = open(filename, "rb")
    a = file.read(2)
    file.close()
    if a == b"\x1f\x8b":
        file = gzip.open(filename, "rt", encoding="latin-1")
    else:
        file = open(filename, "rt", encoding="latin-1")

    try:
        yield file
    finally:
        file.close()


class Timer(object):
    def __init__(self, name=None):
        self.name = name

    def __enter__(self):
        self.tstart = time.time()

    def __exit__(self, type, value, traceback):
        if self.name:
            print("[{}]".format(self.name))
        print("Elapsed: {}".format(time.time() - self.tstart))


def drop_tables(postfix):
    """Drop tables for log file import."""

    db.session.execute("""
        DROP TABLE IF EXISTS "aircraft_beacons_{postfix}";
        DROP TABLE IF EXISTS "receiver_beacons_{postfix}";
    """.format(postfix=postfix))
    db.session.commit()


def create_tables(postfix):
    """Create tables for log file import."""

    drop_tables(postfix)
    db.session.execute("""
        CREATE TABLE aircraft_beacons_{postfix} AS TABLE aircraft_beacons WITH NO DATA;
        CREATE TABLE receiver_beacons_{postfix} AS TABLE receiver_beacons WITH NO DATA;
    """.format(postfix=postfix))
    db.session.commit()


def update_aircraft_beacons_bigdata(postfix):
    """Calculates distance/radial and quality and computes the altitude above ground level.
       Due to performance reasons we use a new table instead of updating the old."""

    db.session.execute("""
        SELECT
            ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed,

            ab.address_type, ab.aircraft_type, ab.stealth, ab.address, ab.climb_rate, ab.turn_rate, ab.signal_quality, ab.error_count,
            ab.frequency_offset, ab.gps_quality_horizontal, ab.gps_quality_vertical, ab.software_version, ab.hardware_version, ab.real_address, ab.signal_power,

            ab.location_mgrs,
            ab.location_mgrs_short,

            CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END AS distance,
            CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) % 360 ELSE NULL END AS radial,
            CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL
                 THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL)
                 ELSE NULL
            END AS quality,
            CAST((ab.altitude - subtable.elev_m) AS REAL) AS agl
        INTO aircraft_beacons_{postfix}_temp
        FROM
            aircraft_beacons_{postfix} AS ab
                JOIN LATERAL (
                    SELECT ab.location, MAX(ST_NearestValue(e.rast, ab.location)) as elev_m
                    FROM elevation e
                    WHERE ST_Intersects(ab.location, e.rast)
                    GROUP BY ab.location
                ) AS subtable ON TRUE,
            (SELECT name, last(location, timestamp) AS location FROM receiver_beacons_{postfix} GROUP BY name) AS r
        WHERE ab.receiver_name = r.name;

        DROP TABLE IF EXISTS "aircraft_beacons_{postfix}";
        ALTER TABLE "aircraft_beacons_{postfix}_temp" RENAME TO "aircraft_beacons_{postfix}";
    """.format(postfix=postfix))


def export_to_path(postfix, path):
    connection = db.engine.raw_connection()
    cursor = connection.cursor()

    aircraft_beacons_file = os.path.join(path, "aircraft_beacons_{postfix}.csv.gz".format(postfix=postfix))
    with gzip.open(aircraft_beacons_file, "wt", encoding="utf-8") as gzip_file:
        cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM aircraft_beacons_{postfix}".format(postfix=postfix)), gzip_file)

    receiver_beacons_file = os.path.join(path, "receiver_beacons_{postfix}.csv.gz".format(postfix=postfix))
    with gzip.open(receiver_beacons_file, "wt") as gzip_file:
        cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM receiver_beacons_{postfix}".format(postfix=postfix)), gzip_file)