Refactoring

ogn/collect/database.py

@@ -1,6 +1,6 @@
 from celery.utils.log import get_task_logger
 
-from sqlalchemy import insert, distinct
+from sqlalchemy import distinct
 from sqlalchemy.sql import null, and_, func, not_, case
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.dialects.postgresql import insert
@@ -29,7 +29,7 @@ def upsert(session, model, rows, update_cols):
     on_conflict_stmt = stmt.on_conflict_do_update(
         index_elements=table.primary_key.columns,
         set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols},
-        )
+    )
 
     # print(compile_query(on_conflict_stmt))
     session.execute(on_conflict_stmt)

ogn/collect/ognrange.py

@@ -1,9 +1,8 @@
 from celery.utils.log import get_task_logger
 
-from sqlalchemy import String, Date
+from sqlalchemy import Date
 from sqlalchemy import and_, insert, update, exists, between
 from sqlalchemy.sql import func, null
-from sqlalchemy.sql.expression import literal_column
 
 from ogn.collect.celery import app
 from ogn.model import AircraftBeacon, ReceiverCoverage

ogn/commands/bulkimport.py

@@ -41,7 +41,7 @@ class LogfileDbSaver():
     def get_datestrs(self, no_index_only=False):
         """Get the date strings from imported log files."""
 
-        index_clause = " AND hasindexes = FALSE" if no_index_only == True else ""
+        index_clause = " AND hasindexes = FALSE" if no_index_only else ""
 
         self.cur.execute(("""
             SELECT DISTINCT(RIGHT(tablename, 10))
@@ -53,81 +53,13 @@ class LogfileDbSaver():
         return [datestr[0].replace('_', '-') for datestr in self.cur.fetchall()]
 
     def create_tables(self):
-        """Create date dependant tables for log file import."""
+        """Create date dependent tables for log file import."""
 
         try:
             self.cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
             self.cur.execute('CREATE EXTENSION IF NOT EXISTS btree_gist;')
-            self.cur.execute('DROP TABLE IF EXISTS "{}";'.format(self.aircraft_table))
-            self.cur.execute('DROP TABLE IF EXISTS "{}";'.format(self.receiver_table))
-            self.cur.execute("""
-                CREATE TABLE "{0}" (
-                    location geometry,
-                    altitude real,
-                    name character varying,
-                    dstcall character varying,
-                    relay character varying,
-                    receiver_name character varying(9),
-                    "timestamp" timestamp without time zone,
-                    track smallint,
-                    ground_speed real,
-
-                    address_type smallint,
-                    aircraft_type smallint,
-                    stealth boolean,
-                    address character varying,
-                    climb_rate real,
-                    turn_rate real,
-                    signal_quality real,
-                    error_count smallint,
-                    frequency_offset real,
-                    gps_quality_horizontal smallint,
-                    gps_quality_vertical smallint,
-                    software_version real,
-                    hardware_version smallint,
-                    real_address character varying(6),
-                    signal_power real,
-
-                    distance real,
-                    radial smallint,
-                    quality real,
-                    location_mgrs character varying(15),
-                    location_mgrs_short character varying(9),
-
-                    receiver_id int,
-                    device_id int);
-            """.format(self.aircraft_table))
-
-            self.cur.execute("""
-                CREATE TABLE "{0}" (
-                    location geometry,
-                    altitude real,
-                    name character varying,
-                    receiver_name character varying(9),
-                    dstcall character varying,
-                    "timestamp" timestamp without time zone,
-
-                    version character varying,
-                    platform character varying,
-                    cpu_load real,
-                    free_ram real,
-                    total_ram real,
-                    ntp_error real,
-                    rt_crystal_correction real,
-                    voltage real,
-                    amperage real,
-                    cpu_temp real,
-                    senders_visible integer,
-                    senders_total integer,
-                    rec_input_noise real,
-                    senders_signal real,
-                    senders_messages integer,
-                    good_senders_signal real,
-                    good_senders integer,
-                    good_and_bad_senders integer,
-
-                    receiver_id int);
-            """.format(self.receiver_table))
+            self.cur.execute('DROP TABLE IF EXISTS "{}"; SELECT INTO {} FROM aircraft_beacons WHERE 1 = 2;'.format(self.aircraft_table))
+            self.cur.execute('DROP TABLE IF EXISTS "{}"; SELECT INTO {} FROM receiver_beacons WHERE 1 = 2;'.format(self.receiver_table))
             self.conn.commit()
         except Exception as e:
             raise Exception("I can't create the tables")

ogn/commands/stats.py

@@ -54,4 +54,4 @@ def create_ognrange(start=None, end=None):
     pbar = tqdm(days)
     for single_date in pbar:
         pbar.set_description(datetime.strftime(single_date, '%Y-%m-%d'))
-        result = create_receiver_coverage(session=session, date=single_date)
+        result = create_receiver_coverage(session=session, date=single_date)

ogn/gateway/process.py

@@ -69,5 +69,3 @@ def process_raw_message(raw_message, reference_date=None, saver=saver):
     logger.debug('Received message: {}'.format(raw_message))
     message = string_to_message(raw_message, reference_date)
     saver.add_message(message)
-
-

ogn/model/aircraft_beacon.py

@@ -28,9 +28,9 @@ class AircraftBeacon(Beacon):
     # Calculated values
     distance = Column(Float(precision=2))
     radial = Column(SmallInteger)
-    quality = Column(Float(precision=2))    # signal quality normalized to 10km
-    location_mgrs = Column(String(15))      # full mgrs (15 chars)
-    location_mgrs_short = Column(String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool
+    quality = Column(Float(precision=2))        # signal quality normalized to 10km
+    location_mgrs = Column(String(15))          # full mgrs (15 chars)
+    location_mgrs_short = Column(String(9))     # reduced mgrs (9 chars), e.g. used for melissas range tool
     agl = Column(Float(precision=2))
 
     # Relations

ogn/model/beacon.py

@@ -1,7 +1,6 @@
 from geoalchemy2.shape import to_shape
 from geoalchemy2.types import Geometry
 from sqlalchemy import Column, String, SmallInteger, Float, DateTime
-from sqlalchemy.sql import func
 from sqlalchemy.ext.declarative import AbstractConcreteBase
 from sqlalchemy.ext.hybrid import hybrid_property
 
@@ -30,8 +29,8 @@ class Beacon(AbstractConcreteBase, Base):
     aprs_type = None
 
     # Debug information
-    raw_message = None #Column(String)
-    reference_timestamp = None #Column(DateTime, index=True)
+    raw_message = None
+    reference_timestamp = None
 
     @hybrid_property
     def location(self):

ogn/model/flights2d.py

@@ -11,7 +11,7 @@ class Flight2D(Base):
     date = Column(Date, primary_key=True)
 
     path_wkt = Column('path', Geometry('MULTILINESTRING', srid=4326))
-    path_simple_wkt = Column('path_simple', Geometry('MULTILINESTRING', srid=4326)) # this is the path simplified with ST_Simplify(path, 0.0001)
+    path_simple_wkt = Column('path_simple', Geometry('MULTILINESTRING', srid=4326))     # this is the path simplified with ST_Simplify(path, 0.0001)
 
     # Relations
     device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), primary_key=True)

ogn/model/receiver_coverage.py

@@ -1,7 +1,5 @@
 from sqlalchemy import Column, String, Integer, SmallInteger, Float, Date, ForeignKey, Index
 from sqlalchemy.orm import relationship, backref
-from sqlalchemy.sql import func
-
 
 from .base import Base