kopia lustrzana https://github.com/glidernet/ogn-python
111 wiersze
4.5 KiB
Python
111 wiersze
4.5 KiB
Python
import os
|
|
import gzip
|
|
import time
|
|
from contextlib import contextmanager
|
|
|
|
from flask import current_app
|
|
from app import db
|
|
|
|
|
|
@contextmanager
|
|
def open_file(filename):
|
|
"""Opens a regular OR gzipped textfile for reading."""
|
|
|
|
file = open(filename, "rb")
|
|
a = file.read(2)
|
|
file.close()
|
|
if a == b"\x1f\x8b":
|
|
file = gzip.open(filename, "rt", encoding="latin-1")
|
|
else:
|
|
file = open(filename, "rt", encoding="latin-1")
|
|
|
|
try:
|
|
yield file
|
|
finally:
|
|
file.close()
|
|
|
|
|
|
class Timer(object):
|
|
def __init__(self, name=None):
|
|
self.name = name
|
|
|
|
def __enter__(self):
|
|
self.tstart = time.time()
|
|
|
|
def __exit__(self, type, value, traceback):
|
|
if self.name:
|
|
print("[{}]".format(self.name))
|
|
print("Elapsed: {}".format(time.time() - self.tstart))
|
|
|
|
|
|
def drop_tables(postfix):
|
|
"""Drop tables for log file import."""
|
|
|
|
db.session.execute("""
|
|
DROP TABLE IF EXISTS "aircraft_beacons_{postfix}";
|
|
DROP TABLE IF EXISTS "receiver_beacons_{postfix}";
|
|
""".format(postfix=postfix))
|
|
db.session.commit()
|
|
|
|
|
|
def create_tables(postfix):
|
|
"""Create tables for log file import."""
|
|
|
|
drop_tables(postfix)
|
|
db.session.execute("""
|
|
CREATE TABLE aircraft_beacons_{postfix} AS TABLE aircraft_beacons WITH NO DATA;
|
|
CREATE TABLE receiver_beacons_{postfix} AS TABLE receiver_beacons WITH NO DATA;
|
|
""".format(postfix=postfix))
|
|
db.session.commit()
|
|
|
|
|
|
def update_aircraft_beacons_bigdata(postfix):
|
|
"""Calculates distance/radial and quality and computes the altitude above ground level.
|
|
Due to performance reasons we use a new table instead of updating the old."""
|
|
|
|
db.session.execute("""
|
|
SELECT
|
|
ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed,
|
|
|
|
ab.address_type, ab.aircraft_type, ab.stealth, ab.address, ab.climb_rate, ab.turn_rate, ab.signal_quality, ab.error_count,
|
|
ab.frequency_offset, ab.gps_quality_horizontal, ab.gps_quality_vertical, ab.software_version, ab.hardware_version, ab.real_address, ab.signal_power,
|
|
|
|
ab.location_mgrs,
|
|
ab.location_mgrs_short,
|
|
|
|
CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END AS distance,
|
|
CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) % 360 ELSE NULL END AS radial,
|
|
CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL
|
|
THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL)
|
|
ELSE NULL
|
|
END AS quality,
|
|
CAST((ab.altitude - subtable.elev_m) AS REAL) AS agl
|
|
INTO aircraft_beacons_{postfix}_temp
|
|
FROM
|
|
aircraft_beacons_{postfix} AS ab
|
|
JOIN LATERAL (
|
|
SELECT ab.location, MAX(ST_NearestValue(e.rast, ab.location)) as elev_m
|
|
FROM elevation e
|
|
WHERE ST_Intersects(ab.location, e.rast)
|
|
GROUP BY ab.location
|
|
) AS subtable ON TRUE,
|
|
(SELECT name, last(location, timestamp) AS location FROM receiver_beacons_{postfix} GROUP BY name) AS r
|
|
WHERE ab.receiver_name = r.name;
|
|
|
|
DROP TABLE IF EXISTS "aircraft_beacons_{postfix}";
|
|
ALTER TABLE "aircraft_beacons_{postfix}_temp" RENAME TO "aircraft_beacons_{postfix}";
|
|
""".format(postfix=postfix))
|
|
|
|
|
|
def export_to_path(postfix, path):
|
|
connection = db.engine.raw_connection()
|
|
cursor = connection.cursor()
|
|
|
|
aircraft_beacons_file = os.path.join(path, "aircraft_beacons_{postfix}.csv.gz".format(postfix=postfix))
|
|
with gzip.open(aircraft_beacons_file, "wt", encoding="utf-8") as gzip_file:
|
|
cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM aircraft_beacons_{postfix}".format(postfix=postfix)), gzip_file)
|
|
|
|
receiver_beacons_file = os.path.join(path, "receiver_beacons_{postfix}.csv.gz".format(postfix=postfix))
|
|
with gzip.open(receiver_beacons_file, "wt") as gzip_file:
|
|
cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM receiver_beacons_{postfix}".format(postfix=postfix)), gzip_file)
|