diff --git a/README.md b/README.md index c1a982e..45cb622 100644 --- a/README.md +++ b/README.md @@ -170,9 +170,18 @@ python3 >>>update_takeoff_landings.delay(last_minutes=90) ``` +or directly from command line: + +``` +celery -A celery_app call takeoff_landings +``` + ## Notes for Raspberry Pi For matplotlib we need several apt packages installed: + +``` apt install libatlas3-base libopenjp2-7 libtiff5 +``` ## License Licensed under the [AGPLv3](LICENSE). diff --git a/app/backend/ognrange.py b/app/backend/ognrange.py index cdef59b..37bd4c1 100644 --- a/app/backend/ognrange.py +++ b/app/backend/ognrange.py @@ -1,8 +1,6 @@ import json from datetime import datetime, timedelta -from sqlalchemy import func, case -from sqlalchemy.sql.expression import label from app.model import Receiver, ReceiverCoverage from app import db @@ -26,11 +24,11 @@ def stations2_filtered_pl(start, end): query = ( db.session.query( Receiver.name.label("s"), - label("lt", func.round(func.ST_Y(Receiver.location_wkt) * 10000) / 10000), - label("lg", func.round(func.ST_X(Receiver.location_wkt) * 10000) / 10000), - case([(Receiver.lastseen > last_10_minutes, "U")], else_="D").label("u"), + db.label("lt", db.func.round(db.func.ST_Y(Receiver.location_wkt) * 10000) / 10000), + db.label("lg", db.func.round(db.func.ST_X(Receiver.location_wkt) * 10000) / 10000), + db.case([(Receiver.lastseen > last_10_minutes, "U")], else_="D").label("u"), Receiver.lastseen.label("ut"), - label("v", Receiver.version + "." + Receiver.platform), + db.label("v", Receiver.version + "." + Receiver.platform), ) .order_by(Receiver.lastseen) .filter(db.or_(db.and_(start < Receiver.firstseen, end > Receiver.firstseen), db.and_(start < Receiver.lastseen, end > Receiver.lastseen))) @@ -44,10 +42,10 @@ def stations2_filtered_pl(start, end): def max_tile_mgrs_pl(station, start, end, squares): query = ( - db.session.query(func.right(ReceiverCoverage.location_mgrs_short, 4), func.count(ReceiverCoverage.location_mgrs_short)) + db.session.query(db.func.right(ReceiverCoverage.location_mgrs_short, 4), db.func.count(ReceiverCoverage.location_mgrs_short)) .filter(db.and_(Receiver.id == ReceiverCoverage.receiver_id, Receiver.name == station)) .filter(ReceiverCoverage.location_mgrs_short.like(squares + "%")) - .group_by(func.right(ReceiverCoverage.location_mgrs_short, 4)) + .group_by(db.func.right(ReceiverCoverage.location_mgrs_short, 4)) ) res = {"t": squares, "p": ["{}/{}".format(r[0], r[1]) for r in query.all()]} diff --git a/app/collect/database.py b/app/collect/database.py index acd3fcc..0c5d51b 100644 --- a/app/collect/database.py +++ b/app/collect/database.py @@ -1,4 +1,3 @@ -from sqlalchemy.sql import null, and_, func, case from sqlalchemy.dialects.postgresql import insert from flask import current_app @@ -15,7 +14,7 @@ def upsert(model, rows, update_cols): stmt = insert(table).values(rows) on_conflict_stmt = stmt.on_conflict_do_update( - index_elements=table.primary_key.columns, set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols} + index_elements=table.primary_key.columns, set_={k: db.case([(getattr(stmt.excluded, k) != db.null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols} ) # print(compile_query(on_conflict_stmt)) diff --git a/app/collect/logbook.py b/app/collect/logbook.py index 0fe971f..b213fe4 100644 --- a/app/collect/logbook.py +++ b/app/collect/logbook.py @@ -1,6 +1,3 @@ -from sqlalchemy import and_, or_, insert, update, exists, between -from sqlalchemy.sql import func, null -from sqlalchemy.sql.expression import case, true, false from sqlalchemy.dialects.postgresql import insert # special insert for upsert ("ON CONFLICT ...") from flask import current_app @@ -46,42 +43,42 @@ def update_takeoff_landings(start, end): .distinct(SenderPosition.name, SenderPosition.timestamp) .order_by(SenderPosition.name, SenderPosition.timestamp, SenderPosition.error_count) .filter(SenderPosition.agl <= MAX_EVENT_AGL) - .filter(between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION))) + .filter(db.between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION))) .subquery() ) # make a query with current, previous and next position sq2 = db.session.query( sq.c.name, - func.lag(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_prev"), - func.lead(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_next"), + db.func.lag(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_prev"), + db.func.lead(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_next"), sq.c.timestamp, - func.lag(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_prev"), - func.lead(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_next"), + db.func.lag(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_prev"), + db.func.lead(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_next"), sq.c.location, - func.lag(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_prev"), - func.lead(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_next"), + db.func.lag(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_prev"), + db.func.lead(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_next"), sq.c.track, - func.lag(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_prev"), - func.lead(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_next"), + db.func.lag(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_prev"), + db.func.lead(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_next"), sq.c.ground_speed, - func.lag(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_prev"), - func.lead(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_next"), + db.func.lag(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_prev"), + db.func.lead(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_next"), sq.c.altitude, - func.lag(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_prev"), - func.lead(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_next"), + db.func.lag(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_prev"), + db.func.lead(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_next"), sq.c.climb_rate, - func.lag(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_prev"), - func.lead(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_next"), + db.func.lag(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_prev"), + db.func.lead(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_next"), ).subquery() # consider only positions between start and end and with predecessor and successor and limit distance and duration between points sq3 = ( db.session.query(sq2) - .filter(and_(sq2.c.name_prev != null(), sq2.c.name_next != null())) - .filter(and_(func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS)) + .filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null())) + .filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS)) .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION)) - .filter(between(sq2.c.timestamp, start, end)) + .filter(db.between(sq2.c.timestamp, start, end)) .subquery() ) @@ -89,24 +86,24 @@ def update_takeoff_landings(start, end): sq4 = ( db.session.query( sq3.c.timestamp, - case( + db.case( [ (sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport (sq3.c.ground_speed <= MIN_TAKEOFF_SPEED, sq3.c.location), ] ).label("location"), - case([(sq3.c.ground_speed > MAX_LANDING_SPEED, sq3.c.track), (sq3.c.ground_speed <= MAX_LANDING_SPEED, sq3.c.track_prev)]).label( + db.case([(sq3.c.ground_speed > MAX_LANDING_SPEED, sq3.c.track), (sq3.c.ground_speed <= MAX_LANDING_SPEED, sq3.c.track_prev)]).label( "track" ), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly sq3.c.ground_speed, sq3.c.altitude, - case([(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, True), (sq3.c.ground_speed < MAX_LANDING_SPEED, False)]).label("is_takeoff"), + db.case([(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, True), (sq3.c.ground_speed < MAX_LANDING_SPEED, False)]).label("is_takeoff"), sq3.c.name, ) .filter( - or_( - and_(sq3.c.ground_speed_prev < MIN_TAKEOFF_SPEED, sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.ground_speed_next > MIN_TAKEOFF_SPEED, sq3.c.climb_rate > MIN_TAKEOFF_CLIMB_RATE), # takeoff - and_(sq3.c.ground_speed_prev > MAX_LANDING_SPEED, sq3.c.ground_speed < MAX_LANDING_SPEED, sq3.c.ground_speed_next < MAX_LANDING_SPEED, sq3.c.climb_rate < MAX_LANDING_SINK_RATE), # landing + db.or_( + db.and_(sq3.c.ground_speed_prev < MIN_TAKEOFF_SPEED, sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.ground_speed_next > MIN_TAKEOFF_SPEED, sq3.c.climb_rate > MIN_TAKEOFF_CLIMB_RATE), # takeoff + db.and_(sq3.c.ground_speed_prev > MAX_LANDING_SPEED, sq3.c.ground_speed < MAX_LANDING_SPEED, sq3.c.ground_speed_next < MAX_LANDING_SPEED, sq3.c.climb_rate < MAX_LANDING_SINK_RATE), # landing ) ) .subquery() @@ -115,10 +112,10 @@ def update_takeoff_landings(start, end): # get the sender id instead of the name and consider them if the are near airports ... sq5 = ( db.session.query( - sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, Sender.id.label("sender_id"), Airport.id.label("airport_id"), func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance"), Airport.country_code + sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, Sender.id.label("sender_id"), Airport.id.label("airport_id"), db.func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance"), Airport.country_code ) - .filter(and_(func.ST_Within(sq4.c.location, Airport.border), - between(Airport.style, 2, 5))) + .filter(db.and_(db.func.ST_Within(sq4.c.location, Airport.border), + db.between(Airport.style, 2, 5))) .filter(sq4.c.name == Sender.name) .subquery() ) @@ -163,26 +160,26 @@ def update_logbook(offset_days=None): else: (start, end) = date_to_timestamps(datetime.utcnow().date()) pa = TakeoffLanding.sender_id - wo = and_(TakeoffLanding.sender_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id) + wo = db.and_(TakeoffLanding.sender_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id) # make a query with previous, current and next "takeoff_landing" event, so we can find complete flights sq = ( db.session.query( TakeoffLanding.sender_id, - func.lag(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_prev"), - func.lead(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_next"), + db.func.lag(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_prev"), + db.func.lead(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_next"), TakeoffLanding.timestamp, - func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"), - func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"), + db.func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"), + db.func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"), TakeoffLanding.track, - func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"), - func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"), + db.func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"), + db.func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"), TakeoffLanding.is_takeoff, - func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"), - func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"), + db.func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"), + db.func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"), TakeoffLanding.airport_id, - func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"), - func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next") + db.func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"), + db.func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next") ) #.filter(between(TakeoffLanding.timestamp, start, end)) .subquery() @@ -196,8 +193,8 @@ def update_logbook(offset_days=None): sq.c.track.label("takeoff_track"), sq.c.airport_id.label("takeoff_airport_id") ) - .filter(sq.c.is_takeoff == true()) - .filter(or_(sq.c.is_takeoff_next == true(), sq.c.is_takeoff_next == null())) + .filter(sq.c.is_takeoff == db.true()) + .filter(db.or_(sq.c.is_takeoff_next == db.true(), sq.c.is_takeoff_next == db.null())) .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.takeoff_timestamp==sq.c.timestamp, Logbook.takeoff_airport_id==sq.c.airport_id)).exists()) ) ins = insert(Logbook).from_select( @@ -221,8 +218,8 @@ def update_logbook(offset_days=None): sq.c.track.label("landing_track"), sq.c.airport_id.label("landing_airport_id"), ) - .filter(or_(sq.c.is_takeoff_prev == false(), sq.c.is_takeoff_prev == null())) - .filter(sq.c.is_takeoff == false()) + .filter(db.or_(sq.c.is_takeoff_prev == db.false(), sq.c.is_takeoff_prev == db.null())) + .filter(sq.c.is_takeoff == db.false()) .filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.landing_timestamp==sq.c.timestamp, Logbook.landing_airport_id==sq.c.airport_id)).exists()) ) ins = insert(Logbook).from_select( @@ -249,8 +246,8 @@ def update_logbook(offset_days=None): sq.c.track_next.label("landing_track"), sq.c.airport_id_next.label("landing_airport_id"), ) - .filter(sq.c.is_takeoff == true()) - .filter(sq.c.is_takeoff_next == false()) + .filter(sq.c.is_takeoff == db.true()) + .filter(sq.c.is_takeoff_next == db.false()) .subquery() ) @@ -277,12 +274,12 @@ def update_logbook(offset_days=None): db.session.commit() # update existing landing with takeoff from complete flight - upd = update(Logbook) \ + upd = db.update(Logbook) \ .where(db.and_( Logbook.sender_id==complete_flight_query.c.sender_id, - Logbook.takeoff_timestamp==null(), - Logbook.takeoff_airport_id==null(), - Logbook.landing_timestamp!=null(), + Logbook.takeoff_timestamp==db.null(), + Logbook.takeoff_airport_id==db.null(), + Logbook.landing_timestamp!=db.null(), Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp, Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id )) \ @@ -295,14 +292,14 @@ def update_logbook(offset_days=None): db.session.commit() # update existing takeoff with landing from complete flight - upd = update(Logbook) \ + upd = db.update(Logbook) \ .where(db.and_( Logbook.sender_id==complete_flight_query.c.sender_id, - Logbook.takeoff_timestamp!=null(), + Logbook.takeoff_timestamp!=db.null(), Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp, Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id, - Logbook.landing_timestamp==null(), - Logbook.landing_airport_id==null() + Logbook.landing_timestamp==db.null(), + Logbook.landing_airport_id==db.null() )) \ .values(landing_timestamp=complete_flight_query.c.landing_timestamp, landing_track=complete_flight_query.c.landing_track, @@ -316,60 +313,56 @@ def update_logbook(offset_days=None): -def update_max_altitudes(offset_days=100): - MAX_UPDATES = 100 +def update_max_altitudes(): + MAX_UPDATES = 60 query = f""" - UPDATE logbooks AS l - SET max_altitude = sq2.max_altitude - FROM ( - SELECT - sq.logbook_id, - MAX(sp.altitude) AS max_altitude - FROM - sender_positions AS sp, - ( - SELECT - l.id AS logbook_id, s.name, l.takeoff_timestamp, l.landing_timestamp - FROM logbooks AS l - INNER JOIN senders AS s ON l.sender_id = s.id - WHERE - l.takeoff_timestamp IS NOT NULL and l.landing_timestamp IS NOT NULL - AND l.max_altitude IS NULL - ORDER BY l.takeoff_timestamp - LIMIT {MAX_UPDATES} - ) AS sq - WHERE sq.name = sp.name AND sp.timestamp BETWEEN sq.takeoff_timestamp AND sq.landing_timestamp - GROUP BY sq.logbook_id - ) AS sq2 - WHERE l.id = sq2.logbook_id; + UPDATE logbooks + SET max_altitude = sq2.max_altitude + FROM ( + SELECT sq.logbook_id, MAX(sp.altitude) AS max_altitude + FROM ( + SELECT + l.id AS logbook_id, s.name, l.takeoff_timestamp, l.landing_timestamp + FROM logbooks AS l + INNER JOIN senders AS s ON l.sender_id = s.id + WHERE + l.takeoff_timestamp IS NOT NULL + AND l.landing_timestamp IS NOT NULL + AND l.max_altitude IS NULL + LIMIT 1 + ) AS sq, + sender_positions AS sp + WHERE sp.reference_timestamp BETWEEN sq.takeoff_timestamp AND sq.landing_timestamp + AND sp.name = sq.name + GROUP BY sq.logbook_id + ) AS sq2 + WHERE logbooks.id = sq2.logbook_id; """ - result = db.session.execute(query) - db.session.commit() + update_counter = 0 + for _ in range(MAX_UPDATES): + result = db.session.execute(query) + db.session.commit() - return result.rowcount + return update_counter -def update_max_altitudes_orm(offset_days): +def update_max_altitudes_orm(): """Add max altitudes in logbook when flight is complete (takeoff and landing).""" current_app.logger.info("Update logbook max altitude.") - (start, end) = date_to_timestamps(datetime.today() - timedelta(days=offset_days)) - logbook_entries = ( db.session.query(Logbook.id, Sender.name) - .filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null())) - #.filter(between(Logbook.reference, start, end)) + .filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null())) .filter(Logbook.sender_id == Sender.id) - .limit(10) + .limit(1) .subquery() ) max_altitudes = ( - db.session.query(Logbook.id, func.max(SenderPosition.altitude).label("max_altitude")) - .filter(and_(SenderPosition.name == logbook_entries.c.name, SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp)) - .filter(Logbook.id == logbook_entries.c.id) + db.session.query(logbook_entries.c.id, db.func.max(SenderPosition.altitude).label("max_altitude")) + .filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name)) .group_by(Logbook.id) .subquery() ) @@ -386,5 +379,6 @@ if __name__ == '__main__': app = create_app() with app.app_context(): #result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0)) - result = update_logbook(0) + #result = update_logbook() + result = update_max_altitudes_orm() print(result) diff --git a/app/commands/logbook.py b/app/commands/logbook.py index 6e7daf4..98ce304 100644 --- a/app/commands/logbook.py +++ b/app/commands/logbook.py @@ -4,8 +4,6 @@ import click from datetime import datetime from app.collect.logbook import update_takeoff_landings, update_logbook -from app.model import Airport, Logbook -from sqlalchemy.sql import func from tqdm import tqdm from app.commands.database import get_database_days from app.utils import date_to_timestamps diff --git a/app/model/coverage_statistic.py b/app/model/coverage_statistic.py index 20bfa48..8fa39d9 100644 --- a/app/model/coverage_statistic.py +++ b/app/model/coverage_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - class CoverageStatistic(db.Model): __tablename__ = "coverage_statistics" @@ -20,9 +17,9 @@ class CoverageStatistic(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("coverage_stats", order_by=date)) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("coverage_stats", order_by=date)) receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) - receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("coverage_stats", order_by=date)) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("coverage_stats", order_by=date)) - __table_args__ = (Index('idx_coverage_statistics_uc', 'date', 'location_mgrs_short', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) + __table_args__ = (db.Index('idx_coverage_statistics_uc', 'date', 'location_mgrs_short', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/logbook.py b/app/model/logbook.py index 995a553..748a4ae 100644 --- a/app/model/logbook.py +++ b/app/model/logbook.py @@ -1,6 +1,4 @@ from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.sql import null, case -from sqlalchemy.orm import backref from app import db @@ -18,19 +16,19 @@ class Logbook(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("logbook_entries", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("logbook_entries", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc())) takeoff_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) - takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id], backref=backref("logbook_entries_takeoff", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id], backref=db.backref("logbook_entries_takeoff", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc())) takeoff_country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True) - takeoff_country = db.relationship("Country", foreign_keys=[takeoff_country_id], backref=backref("logbook_entries_takeoff", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + takeoff_country = db.relationship("Country", foreign_keys=[takeoff_country_id], backref=db.backref("logbook_entries_takeoff", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc())) landing_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) - landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id], backref=backref("logbook_entries_landing", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id], backref=db.backref("logbook_entries_landing", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc())) landing_country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True) - landing_country = db.relationship("Country", foreign_keys=[landing_country_id], backref=backref("logbook_entries_landing", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + landing_country = db.relationship("Country", foreign_keys=[landing_country_id], backref=db.backref("logbook_entries_landing", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc())) @hybrid_property def duration(self): @@ -38,7 +36,7 @@ class Logbook(db.Model): @duration.expression def duration(cls): - return case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != null() and cls.takeoff_timestamp != null()) + return db.case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != db.null() and cls.takeoff_timestamp != db.null()) @hybrid_property def reference_timestamp(self): @@ -46,13 +44,13 @@ class Logbook(db.Model): @reference_timestamp.expression def reference_timestamp(cls): - return case({True: cls.takeoff_timestamp, False: cls.landing_timestamp}, cls.takeoff_timestamp != null()) + return db.case({True: cls.takeoff_timestamp, False: cls.landing_timestamp}, cls.takeoff_timestamp != db.null()) - #__table_args__ = (db.Index('idx_logbook_reference_timestamp', case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null())),) + #__table_args__ = (db.Index('idx_logbook_reference_timestamp', db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null())),) # FIXME: does not work... # FIXME: this does not throw an error as the __table_args__ above, but there is no index created -#_wrapped_case = f"({case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != null())})" +#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})" #Index("idx_logbook_reference_timestamp", _wrapped_case) # TODO: diff --git a/app/model/receiver.py b/app/model/receiver.py index 74b1b2b..67f24ee 100644 --- a/app/model/receiver.py +++ b/app/model/receiver.py @@ -4,7 +4,6 @@ from geoalchemy2.types import Geometry from .geo import Location from app import db -from sqlalchemy import Index from .airport import Airport @@ -35,7 +34,7 @@ class Receiver(db.Model): airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) airport = db.relationship("Airport", foreign_keys=[airport_id], backref=db.backref("receivers", order_by="Receiver.name.asc()")) - __table_args__ = (Index('idx_receivers_name_uc', 'name', unique=True), ) + __table_args__ = (db.Index('idx_receivers_name_uc', 'name', unique=True), ) @property def location(self): diff --git a/app/model/receiver_statistic.py b/app/model/receiver_statistic.py index 5bba591..1b20b59 100644 --- a/app/model/receiver_statistic.py +++ b/app/model/receiver_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - class ReceiverStatistic(db.Model): __tablename__ = "receiver_statistics" @@ -20,6 +17,6 @@ class ReceiverStatistic(db.Model): # Relations receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) - receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("statistics", order_by=date.desc())) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("statistics", order_by=date.desc())) - __table_args__ = (Index('idx_receiver_statistics_uc', 'date', 'receiver_id', 'is_trustworthy', unique=True), ) + __table_args__ = (db.Index('idx_receiver_statistics_uc', 'date', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/relation_statistic.py b/app/model/relation_statistic.py index bb8205b..92dfeb6 100644 --- a/app/model/relation_statistic.py +++ b/app/model/relation_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - class RelationStatistic(db.Model): __tablename__ = "relation_statistics" @@ -18,9 +15,9 @@ class RelationStatistic(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("relation_stats", order_by=date)) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("relation_stats", order_by=date)) receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) - receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("relation_stats", order_by=date)) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("relation_stats", order_by=date)) - __table_args__ = (Index('idx_relation_statistics_uc', 'date', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) + __table_args__ = (db.Index('idx_relation_statistics_uc', 'date', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/sender.py b/app/model/sender.py index 18609e5..7273cb6 100644 --- a/app/model/sender.py +++ b/app/model/sender.py @@ -1,7 +1,6 @@ import datetime from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy import Index from app import db from app.model.aircraft_type import AircraftType @@ -22,7 +21,7 @@ class Sender(db.Model): hardware_version = db.Column(db.SmallInteger) real_address = db.Column(db.String(6)) - __table_args__ = (Index('idx_senders_name_uc', 'name', unique=True), ) + __table_args__ = (db.Index('idx_senders_name_uc', 'name', unique=True), ) def __repr__(self): return "" % (self.address, self.aircraft_type, self.stealth, self.software_version, self.hardware_version, self.real_address) diff --git a/app/model/sender_direction_statistic.py b/app/model/sender_direction_statistic.py index 5ccc63f..96b759a 100644 --- a/app/model/sender_direction_statistic.py +++ b/app/model/sender_direction_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - from sqlalchemy.dialects.postgresql import JSON class SenderDirectionStatistic(db.Model): @@ -16,9 +13,9 @@ class SenderDirectionStatistic(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("direction_stats", order_by=directions_count.desc())) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("direction_stats", order_by=directions_count.desc())) receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) - receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("direction_stats", order_by=directions_count.desc())) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("direction_stats", order_by=directions_count.desc())) - __table_args__ = (Index('idx_sender_direction_statistics_uc', 'sender_id', 'receiver_id', unique=True), ) + __table_args__ = (db.Index('idx_sender_direction_statistics_uc', 'sender_id', 'receiver_id', unique=True), ) diff --git a/app/model/sender_info.py b/app/model/sender_info.py index 67aa303..8c067b1 100644 --- a/app/model/sender_info.py +++ b/app/model/sender_info.py @@ -2,7 +2,6 @@ from app import db from .sender_info_origin import SenderInfoOrigin from .aircraft_type import AircraftType -from sqlalchemy.orm import backref #from sqlalchemy.dialects.postgresql import ENUM class SenderInfo(db.Model): @@ -22,7 +21,7 @@ class SenderInfo(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("infos", order_by=address_origin)) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("infos", order_by=address_origin)) def __repr__(self): return "" % ( diff --git a/app/model/sender_position_statistic.py b/app/model/sender_position_statistic.py index 6fcf497..a433227 100644 --- a/app/model/sender_position_statistic.py +++ b/app/model/sender_position_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - from .aircraft_type import AircraftType from sqlalchemy.dialects.postgresql import ENUM @@ -23,4 +20,4 @@ class SenderPositionStatistic(db.Model): messages_count = db.Column(db.Integer) - __table_args__ = (Index('idx_sender_position_statistics_uc', 'date', 'dstcall', 'address_type', 'aircraft_type', 'stealth', 'software_version', 'hardware_version', unique=True), ) + __table_args__ = (db.Index('idx_sender_position_statistics_uc', 'date', 'dstcall', 'address_type', 'aircraft_type', 'stealth', 'software_version', 'hardware_version', unique=True), ) diff --git a/app/model/sender_statistic.py b/app/model/sender_statistic.py index 392d94a..9721683 100644 --- a/app/model/sender_statistic.py +++ b/app/model/sender_statistic.py @@ -1,8 +1,5 @@ from app import db -from sqlalchemy import Index -from sqlalchemy.orm import backref - class SenderStatistic(db.Model): __tablename__ = "sender_statistics" @@ -20,6 +17,6 @@ class SenderStatistic(db.Model): # Relations sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) - sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("statistics", order_by=date.desc())) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("statistics", order_by=date.desc())) - __table_args__ = (Index('idx_sender_statistics_uc', 'date', 'sender_id', 'is_trustworthy', unique=True), ) + __table_args__ = (db.Index('idx_sender_statistics_uc', 'date', 'sender_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/takeoff_landing.py b/app/model/takeoff_landing.py index 66b27a2..0598ca2 100644 --- a/app/model/takeoff_landing.py +++ b/app/model/takeoff_landing.py @@ -1,5 +1,4 @@ from app import db -from sqlalchemy import Index class TakeoffLanding(db.Model): @@ -21,4 +20,4 @@ class TakeoffLanding(db.Model): country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True) country = db.relationship("Country", foreign_keys=[country_id], backref="takeoff_landings") - __table_args__ = (Index('idx_takeoff_landings_uc', 'timestamp', 'sender_id', 'airport_id', unique=True), ) + __table_args__ = (db.Index('idx_takeoff_landings_uc', 'timestamp', 'sender_id', 'airport_id', unique=True), ) diff --git a/app/tasks/orm_tasks.py b/app/tasks/orm_tasks.py index b308f0c..125d3f9 100644 --- a/app/tasks/orm_tasks.py +++ b/app/tasks/orm_tasks.py @@ -36,10 +36,10 @@ def update_logbook(offset_days=None): @celery.task(name="update_logbook_max_altitude") -def update_logbook_max_altitude(offset_days=0): +def update_logbook_max_altitude(): """Add max altitudes in logbook when flight is complete (takeoff and landing).""" - result = logbook_update_max_altitudes(offset_days=offset_days) + result = logbook_update_max_altitudes() return result diff --git a/app/templates/receivers.html b/app/templates/receivers.html index 2e4d70d..23a8933 100644 --- a/app/templates/receivers.html +++ b/app/templates/receivers.html @@ -36,8 +36,8 @@ {{ loop.index }} {{ receiver.country.iso2 }} - {{ receiver.name }} - {{ receiver.airport.name }} + {{ receiver|to_html_link|safe }} + {{ receiver.airport|to_html_link|safe }} {{ receiver.altitude|int }} m {{ receiver.lastseen|timestamp_to_status|safe }} {{ receiver.version if receiver.version else '-' }} diff --git a/app/templates/sender_ranking.html b/app/templates/sender_ranking.html index 174d91e..9113478 100644 --- a/app/templates/sender_ranking.html +++ b/app/templates/sender_ranking.html @@ -21,7 +21,7 @@ {{ loop.index }} {{ entry.sender|to_html_link|safe }} - {% if entry.sender.infos|length > 0 %}{{ entry.sender.infos[0].aircraft }}{% endif %} + {% if entry.sender.infos|length > 0 %}{{ entry.sender.infos[0].aircraft }}{% else %}-{% endif %} {{ '%0.1f' | format(entry.max_distance/1000.0) }} {{ '%0.1f' | format(entry.max_normalized_quality) }} {{ entry.receivers_count }}