kopia lustrzana https://github.com/glidernet/ogn-python
Refactoring
Konstantin Gründger
rodzic
5234107606
commit
4c157bff4a
|
|
@ -170,9 +170,18 @@ python3
|
|||
>>>update_takeoff_landings.delay(last_minutes=90)
|
||||
```
|
||||
|
||||
or directly from command line:
|
||||
|
||||
```
|
||||
celery -A celery_app call takeoff_landings
|
||||
```
|
||||
|
||||
## Notes for Raspberry Pi
|
||||
For matplotlib we need several apt packages installed:
|
||||
|
||||
```
|
||||
apt install libatlas3-base libopenjp2-7 libtiff5
|
||||
```
|
||||
|
||||
## License
|
||||
Licensed under the [AGPLv3](LICENSE).
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import func, case
|
||||
from sqlalchemy.sql.expression import label
|
||||
from app.model import Receiver, ReceiverCoverage
|
||||
|
||||
from app import db
|
||||
|
|
@ -26,11 +24,11 @@ def stations2_filtered_pl(start, end):
|
|||
query = (
|
||||
db.session.query(
|
||||
Receiver.name.label("s"),
|
||||
label("lt", func.round(func.ST_Y(Receiver.location_wkt) * 10000) / 10000),
|
||||
label("lg", func.round(func.ST_X(Receiver.location_wkt) * 10000) / 10000),
|
||||
case([(Receiver.lastseen > last_10_minutes, "U")], else_="D").label("u"),
|
||||
db.label("lt", db.func.round(db.func.ST_Y(Receiver.location_wkt) * 10000) / 10000),
|
||||
db.label("lg", db.func.round(db.func.ST_X(Receiver.location_wkt) * 10000) / 10000),
|
||||
db.case([(Receiver.lastseen > last_10_minutes, "U")], else_="D").label("u"),
|
||||
Receiver.lastseen.label("ut"),
|
||||
label("v", Receiver.version + "." + Receiver.platform),
|
||||
db.label("v", Receiver.version + "." + Receiver.platform),
|
||||
)
|
||||
.order_by(Receiver.lastseen)
|
||||
.filter(db.or_(db.and_(start < Receiver.firstseen, end > Receiver.firstseen), db.and_(start < Receiver.lastseen, end > Receiver.lastseen)))
|
||||
|
|
@ -44,10 +42,10 @@ def stations2_filtered_pl(start, end):
|
|||
|
||||
def max_tile_mgrs_pl(station, start, end, squares):
|
||||
query = (
|
||||
db.session.query(func.right(ReceiverCoverage.location_mgrs_short, 4), func.count(ReceiverCoverage.location_mgrs_short))
|
||||
db.session.query(db.func.right(ReceiverCoverage.location_mgrs_short, 4), db.func.count(ReceiverCoverage.location_mgrs_short))
|
||||
.filter(db.and_(Receiver.id == ReceiverCoverage.receiver_id, Receiver.name == station))
|
||||
.filter(ReceiverCoverage.location_mgrs_short.like(squares + "%"))
|
||||
.group_by(func.right(ReceiverCoverage.location_mgrs_short, 4))
|
||||
.group_by(db.func.right(ReceiverCoverage.location_mgrs_short, 4))
|
||||
)
|
||||
|
||||
res = {"t": squares, "p": ["{}/{}".format(r[0], r[1]) for r in query.all()]}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from sqlalchemy.sql import null, and_, func, case
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from flask import current_app
|
||||
|
||||
|
|
@ -15,7 +14,7 @@ def upsert(model, rows, update_cols):
|
|||
stmt = insert(table).values(rows)
|
||||
|
||||
on_conflict_stmt = stmt.on_conflict_do_update(
|
||||
index_elements=table.primary_key.columns, set_={k: case([(getattr(stmt.excluded, k) != null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols}
|
||||
index_elements=table.primary_key.columns, set_={k: db.case([(getattr(stmt.excluded, k) != db.null(), getattr(stmt.excluded, k))], else_=getattr(model, k)) for k in update_cols}
|
||||
)
|
||||
|
||||
# print(compile_query(on_conflict_stmt))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
from sqlalchemy import and_, or_, insert, update, exists, between
|
||||
from sqlalchemy.sql import func, null
|
||||
from sqlalchemy.sql.expression import case, true, false
|
||||
from sqlalchemy.dialects.postgresql import insert # special insert for upsert ("ON CONFLICT ...")
|
||||
from flask import current_app
|
||||
|
||||
|
|
@ -46,42 +43,42 @@ def update_takeoff_landings(start, end):
|
|||
.distinct(SenderPosition.name, SenderPosition.timestamp)
|
||||
.order_by(SenderPosition.name, SenderPosition.timestamp, SenderPosition.error_count)
|
||||
.filter(SenderPosition.agl <= MAX_EVENT_AGL)
|
||||
.filter(between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION)))
|
||||
.filter(db.between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION)))
|
||||
.subquery()
|
||||
)
|
||||
|
||||
# make a query with current, previous and next position
|
||||
sq2 = db.session.query(
|
||||
sq.c.name,
|
||||
func.lag(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_prev"),
|
||||
func.lead(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_next"),
|
||||
db.func.lag(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_prev"),
|
||||
db.func.lead(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_next"),
|
||||
sq.c.timestamp,
|
||||
func.lag(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_prev"),
|
||||
func.lead(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_next"),
|
||||
db.func.lag(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_prev"),
|
||||
db.func.lead(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_next"),
|
||||
sq.c.location,
|
||||
func.lag(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_prev"),
|
||||
func.lead(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_next"),
|
||||
db.func.lag(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_prev"),
|
||||
db.func.lead(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_next"),
|
||||
sq.c.track,
|
||||
func.lag(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_prev"),
|
||||
func.lead(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_next"),
|
||||
db.func.lag(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_prev"),
|
||||
db.func.lead(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_next"),
|
||||
sq.c.ground_speed,
|
||||
func.lag(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_prev"),
|
||||
func.lead(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_next"),
|
||||
db.func.lag(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_prev"),
|
||||
db.func.lead(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_next"),
|
||||
sq.c.altitude,
|
||||
func.lag(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_prev"),
|
||||
func.lead(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_next"),
|
||||
db.func.lag(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_prev"),
|
||||
db.func.lead(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_next"),
|
||||
sq.c.climb_rate,
|
||||
func.lag(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_prev"),
|
||||
func.lead(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_next"),
|
||||
db.func.lag(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_prev"),
|
||||
db.func.lead(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_next"),
|
||||
).subquery()
|
||||
|
||||
# consider only positions between start and end and with predecessor and successor and limit distance and duration between points
|
||||
sq3 = (
|
||||
db.session.query(sq2)
|
||||
.filter(and_(sq2.c.name_prev != null(), sq2.c.name_next != null()))
|
||||
.filter(and_(func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS))
|
||||
.filter(db.and_(sq2.c.name_prev != db.null(), sq2.c.name_next != db.null()))
|
||||
.filter(db.and_(db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, db.func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS))
|
||||
.filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION))
|
||||
.filter(between(sq2.c.timestamp, start, end))
|
||||
.filter(db.between(sq2.c.timestamp, start, end))
|
||||
.subquery()
|
||||
)
|
||||
|
||||
|
|
@ -89,24 +86,24 @@ def update_takeoff_landings(start, end):
|
|||
sq4 = (
|
||||
db.session.query(
|
||||
sq3.c.timestamp,
|
||||
case(
|
||||
db.case(
|
||||
[
|
||||
(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport
|
||||
(sq3.c.ground_speed <= MIN_TAKEOFF_SPEED, sq3.c.location),
|
||||
]
|
||||
).label("location"),
|
||||
case([(sq3.c.ground_speed > MAX_LANDING_SPEED, sq3.c.track), (sq3.c.ground_speed <= MAX_LANDING_SPEED, sq3.c.track_prev)]).label(
|
||||
db.case([(sq3.c.ground_speed > MAX_LANDING_SPEED, sq3.c.track), (sq3.c.ground_speed <= MAX_LANDING_SPEED, sq3.c.track_prev)]).label(
|
||||
"track"
|
||||
), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly
|
||||
sq3.c.ground_speed,
|
||||
sq3.c.altitude,
|
||||
case([(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, True), (sq3.c.ground_speed < MAX_LANDING_SPEED, False)]).label("is_takeoff"),
|
||||
db.case([(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, True), (sq3.c.ground_speed < MAX_LANDING_SPEED, False)]).label("is_takeoff"),
|
||||
sq3.c.name,
|
||||
)
|
||||
.filter(
|
||||
or_(
|
||||
and_(sq3.c.ground_speed_prev < MIN_TAKEOFF_SPEED, sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.ground_speed_next > MIN_TAKEOFF_SPEED, sq3.c.climb_rate > MIN_TAKEOFF_CLIMB_RATE), # takeoff
|
||||
and_(sq3.c.ground_speed_prev > MAX_LANDING_SPEED, sq3.c.ground_speed < MAX_LANDING_SPEED, sq3.c.ground_speed_next < MAX_LANDING_SPEED, sq3.c.climb_rate < MAX_LANDING_SINK_RATE), # landing
|
||||
db.or_(
|
||||
db.and_(sq3.c.ground_speed_prev < MIN_TAKEOFF_SPEED, sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.ground_speed_next > MIN_TAKEOFF_SPEED, sq3.c.climb_rate > MIN_TAKEOFF_CLIMB_RATE), # takeoff
|
||||
db.and_(sq3.c.ground_speed_prev > MAX_LANDING_SPEED, sq3.c.ground_speed < MAX_LANDING_SPEED, sq3.c.ground_speed_next < MAX_LANDING_SPEED, sq3.c.climb_rate < MAX_LANDING_SINK_RATE), # landing
|
||||
)
|
||||
)
|
||||
.subquery()
|
||||
|
|
@ -115,10 +112,10 @@ def update_takeoff_landings(start, end):
|
|||
# get the sender id instead of the name and consider them if the are near airports ...
|
||||
sq5 = (
|
||||
db.session.query(
|
||||
sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, Sender.id.label("sender_id"), Airport.id.label("airport_id"), func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance"), Airport.country_code
|
||||
sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, Sender.id.label("sender_id"), Airport.id.label("airport_id"), db.func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance"), Airport.country_code
|
||||
)
|
||||
.filter(and_(func.ST_Within(sq4.c.location, Airport.border),
|
||||
between(Airport.style, 2, 5)))
|
||||
.filter(db.and_(db.func.ST_Within(sq4.c.location, Airport.border),
|
||||
db.between(Airport.style, 2, 5)))
|
||||
.filter(sq4.c.name == Sender.name)
|
||||
.subquery()
|
||||
)
|
||||
|
|
@ -163,26 +160,26 @@ def update_logbook(offset_days=None):
|
|||
else:
|
||||
(start, end) = date_to_timestamps(datetime.utcnow().date())
|
||||
pa = TakeoffLanding.sender_id
|
||||
wo = and_(TakeoffLanding.sender_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id)
|
||||
wo = db.and_(TakeoffLanding.sender_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id)
|
||||
|
||||
# make a query with previous, current and next "takeoff_landing" event, so we can find complete flights
|
||||
sq = (
|
||||
db.session.query(
|
||||
TakeoffLanding.sender_id,
|
||||
func.lag(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_prev"),
|
||||
func.lead(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_next"),
|
||||
db.func.lag(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_prev"),
|
||||
db.func.lead(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_next"),
|
||||
TakeoffLanding.timestamp,
|
||||
func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"),
|
||||
func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"),
|
||||
db.func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"),
|
||||
db.func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"),
|
||||
TakeoffLanding.track,
|
||||
func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"),
|
||||
func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"),
|
||||
db.func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"),
|
||||
db.func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"),
|
||||
TakeoffLanding.is_takeoff,
|
||||
func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"),
|
||||
func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"),
|
||||
db.func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"),
|
||||
db.func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"),
|
||||
TakeoffLanding.airport_id,
|
||||
func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"),
|
||||
func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next")
|
||||
db.func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"),
|
||||
db.func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next")
|
||||
)
|
||||
#.filter(between(TakeoffLanding.timestamp, start, end))
|
||||
.subquery()
|
||||
|
|
@ -196,8 +193,8 @@ def update_logbook(offset_days=None):
|
|||
sq.c.track.label("takeoff_track"),
|
||||
sq.c.airport_id.label("takeoff_airport_id")
|
||||
)
|
||||
.filter(sq.c.is_takeoff == true())
|
||||
.filter(or_(sq.c.is_takeoff_next == true(), sq.c.is_takeoff_next == null()))
|
||||
.filter(sq.c.is_takeoff == db.true())
|
||||
.filter(db.or_(sq.c.is_takeoff_next == db.true(), sq.c.is_takeoff_next == db.null()))
|
||||
.filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.takeoff_timestamp==sq.c.timestamp, Logbook.takeoff_airport_id==sq.c.airport_id)).exists())
|
||||
)
|
||||
ins = insert(Logbook).from_select(
|
||||
|
|
@ -221,8 +218,8 @@ def update_logbook(offset_days=None):
|
|||
sq.c.track.label("landing_track"),
|
||||
sq.c.airport_id.label("landing_airport_id"),
|
||||
)
|
||||
.filter(or_(sq.c.is_takeoff_prev == false(), sq.c.is_takeoff_prev == null()))
|
||||
.filter(sq.c.is_takeoff == false())
|
||||
.filter(db.or_(sq.c.is_takeoff_prev == db.false(), sq.c.is_takeoff_prev == db.null()))
|
||||
.filter(sq.c.is_takeoff == db.false())
|
||||
.filter(~Logbook.query.filter(db.and_(Logbook.sender_id==sq.c.sender_id, Logbook.landing_timestamp==sq.c.timestamp, Logbook.landing_airport_id==sq.c.airport_id)).exists())
|
||||
)
|
||||
ins = insert(Logbook).from_select(
|
||||
|
|
@ -249,8 +246,8 @@ def update_logbook(offset_days=None):
|
|||
sq.c.track_next.label("landing_track"),
|
||||
sq.c.airport_id_next.label("landing_airport_id"),
|
||||
)
|
||||
.filter(sq.c.is_takeoff == true())
|
||||
.filter(sq.c.is_takeoff_next == false())
|
||||
.filter(sq.c.is_takeoff == db.true())
|
||||
.filter(sq.c.is_takeoff_next == db.false())
|
||||
.subquery()
|
||||
)
|
||||
|
||||
|
|
@ -277,12 +274,12 @@ def update_logbook(offset_days=None):
|
|||
db.session.commit()
|
||||
|
||||
# update existing landing with takeoff from complete flight
|
||||
upd = update(Logbook) \
|
||||
upd = db.update(Logbook) \
|
||||
.where(db.and_(
|
||||
Logbook.sender_id==complete_flight_query.c.sender_id,
|
||||
Logbook.takeoff_timestamp==null(),
|
||||
Logbook.takeoff_airport_id==null(),
|
||||
Logbook.landing_timestamp!=null(),
|
||||
Logbook.takeoff_timestamp==db.null(),
|
||||
Logbook.takeoff_airport_id==db.null(),
|
||||
Logbook.landing_timestamp!=db.null(),
|
||||
Logbook.landing_timestamp==complete_flight_query.c.landing_timestamp,
|
||||
Logbook.landing_airport_id==complete_flight_query.c.landing_airport_id
|
||||
)) \
|
||||
|
|
@ -295,14 +292,14 @@ def update_logbook(offset_days=None):
|
|||
db.session.commit()
|
||||
|
||||
# update existing takeoff with landing from complete flight
|
||||
upd = update(Logbook) \
|
||||
upd = db.update(Logbook) \
|
||||
.where(db.and_(
|
||||
Logbook.sender_id==complete_flight_query.c.sender_id,
|
||||
Logbook.takeoff_timestamp!=null(),
|
||||
Logbook.takeoff_timestamp!=db.null(),
|
||||
Logbook.takeoff_timestamp==complete_flight_query.c.takeoff_timestamp,
|
||||
Logbook.takeoff_airport_id==complete_flight_query.c.takeoff_airport_id,
|
||||
Logbook.landing_timestamp==null(),
|
||||
Logbook.landing_airport_id==null()
|
||||
Logbook.landing_timestamp==db.null(),
|
||||
Logbook.landing_airport_id==db.null()
|
||||
)) \
|
||||
.values(landing_timestamp=complete_flight_query.c.landing_timestamp,
|
||||
landing_track=complete_flight_query.c.landing_track,
|
||||
|
|
@ -316,60 +313,56 @@ def update_logbook(offset_days=None):
|
|||
|
||||
|
||||
|
||||
def update_max_altitudes(offset_days=100):
|
||||
MAX_UPDATES = 100
|
||||
def update_max_altitudes():
|
||||
MAX_UPDATES = 60
|
||||
|
||||
query = f"""
|
||||
UPDATE logbooks AS l
|
||||
SET max_altitude = sq2.max_altitude
|
||||
FROM (
|
||||
SELECT
|
||||
sq.logbook_id,
|
||||
MAX(sp.altitude) AS max_altitude
|
||||
FROM
|
||||
sender_positions AS sp,
|
||||
(
|
||||
SELECT
|
||||
l.id AS logbook_id, s.name, l.takeoff_timestamp, l.landing_timestamp
|
||||
FROM logbooks AS l
|
||||
INNER JOIN senders AS s ON l.sender_id = s.id
|
||||
WHERE
|
||||
l.takeoff_timestamp IS NOT NULL and l.landing_timestamp IS NOT NULL
|
||||
AND l.max_altitude IS NULL
|
||||
ORDER BY l.takeoff_timestamp
|
||||
LIMIT {MAX_UPDATES}
|
||||
) AS sq
|
||||
WHERE sq.name = sp.name AND sp.timestamp BETWEEN sq.takeoff_timestamp AND sq.landing_timestamp
|
||||
GROUP BY sq.logbook_id
|
||||
) AS sq2
|
||||
WHERE l.id = sq2.logbook_id;
|
||||
UPDATE logbooks
|
||||
SET max_altitude = sq2.max_altitude
|
||||
FROM (
|
||||
SELECT sq.logbook_id, MAX(sp.altitude) AS max_altitude
|
||||
FROM (
|
||||
SELECT
|
||||
l.id AS logbook_id, s.name, l.takeoff_timestamp, l.landing_timestamp
|
||||
FROM logbooks AS l
|
||||
INNER JOIN senders AS s ON l.sender_id = s.id
|
||||
WHERE
|
||||
l.takeoff_timestamp IS NOT NULL
|
||||
AND l.landing_timestamp IS NOT NULL
|
||||
AND l.max_altitude IS NULL
|
||||
LIMIT 1
|
||||
) AS sq,
|
||||
sender_positions AS sp
|
||||
WHERE sp.reference_timestamp BETWEEN sq.takeoff_timestamp AND sq.landing_timestamp
|
||||
AND sp.name = sq.name
|
||||
GROUP BY sq.logbook_id
|
||||
) AS sq2
|
||||
WHERE logbooks.id = sq2.logbook_id;
|
||||
"""
|
||||
|
||||
result = db.session.execute(query)
|
||||
db.session.commit()
|
||||
update_counter = 0
|
||||
for _ in range(MAX_UPDATES):
|
||||
result = db.session.execute(query)
|
||||
db.session.commit()
|
||||
|
||||
return result.rowcount
|
||||
return update_counter
|
||||
|
||||
def update_max_altitudes_orm(offset_days):
|
||||
def update_max_altitudes_orm():
|
||||
"""Add max altitudes in logbook when flight is complete (takeoff and landing)."""
|
||||
|
||||
current_app.logger.info("Update logbook max altitude.")
|
||||
|
||||
(start, end) = date_to_timestamps(datetime.today() - timedelta(days=offset_days))
|
||||
|
||||
logbook_entries = (
|
||||
db.session.query(Logbook.id, Sender.name)
|
||||
.filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null()))
|
||||
#.filter(between(Logbook.reference, start, end))
|
||||
.filter(db.and_(Logbook.takeoff_timestamp != db.null(), Logbook.landing_timestamp != db.null(), Logbook.max_altitude == db.null()))
|
||||
.filter(Logbook.sender_id == Sender.id)
|
||||
.limit(10)
|
||||
.limit(1)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
max_altitudes = (
|
||||
db.session.query(Logbook.id, func.max(SenderPosition.altitude).label("max_altitude"))
|
||||
.filter(and_(SenderPosition.name == logbook_entries.c.name, SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp))
|
||||
.filter(Logbook.id == logbook_entries.c.id)
|
||||
db.session.query(logbook_entries.c.id, db.func.max(SenderPosition.altitude).label("max_altitude"))
|
||||
.filter(db.and_(db.between_(SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp), SenderPosition.name == logbook_entries.c.name))
|
||||
.group_by(Logbook.id)
|
||||
.subquery()
|
||||
)
|
||||
|
|
@ -386,5 +379,6 @@ if __name__ == '__main__':
|
|||
app = create_app()
|
||||
with app.app_context():
|
||||
#result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 15, 30, 0))
|
||||
result = update_logbook(0)
|
||||
#result = update_logbook()
|
||||
result = update_max_altitudes_orm()
|
||||
print(result)
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ import click
|
|||
from datetime import datetime
|
||||
|
||||
from app.collect.logbook import update_takeoff_landings, update_logbook
|
||||
from app.model import Airport, Logbook
|
||||
from sqlalchemy.sql import func
|
||||
from tqdm import tqdm
|
||||
from app.commands.database import get_database_days
|
||||
from app.utils import date_to_timestamps
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
|
||||
class CoverageStatistic(db.Model):
|
||||
__tablename__ = "coverage_statistics"
|
||||
|
|
@ -20,9 +17,9 @@ class CoverageStatistic(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("coverage_stats", order_by=date))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("coverage_stats", order_by=date))
|
||||
|
||||
receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True)
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("coverage_stats", order_by=date))
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("coverage_stats", order_by=date))
|
||||
|
||||
__table_args__ = (Index('idx_coverage_statistics_uc', 'date', 'location_mgrs_short', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
__table_args__ = (db.Index('idx_coverage_statistics_uc', 'date', 'location_mgrs_short', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from sqlalchemy.ext.hybrid import hybrid_property
|
||||
from sqlalchemy.sql import null, case
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
from app import db
|
||||
|
||||
|
|
@ -18,19 +16,19 @@ class Logbook(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("logbook_entries", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc()))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("logbook_entries", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc()))
|
||||
|
||||
takeoff_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True)
|
||||
takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id], backref=backref("logbook_entries_takeoff", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc()))
|
||||
takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id], backref=db.backref("logbook_entries_takeoff", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc()))
|
||||
|
||||
takeoff_country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True)
|
||||
takeoff_country = db.relationship("Country", foreign_keys=[takeoff_country_id], backref=backref("logbook_entries_takeoff", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc()))
|
||||
takeoff_country = db.relationship("Country", foreign_keys=[takeoff_country_id], backref=db.backref("logbook_entries_takeoff", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc()))
|
||||
|
||||
landing_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True)
|
||||
landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id], backref=backref("logbook_entries_landing", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc()))
|
||||
landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id], backref=db.backref("logbook_entries_landing", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc()))
|
||||
|
||||
landing_country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True)
|
||||
landing_country = db.relationship("Country", foreign_keys=[landing_country_id], backref=backref("logbook_entries_landing", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc()))
|
||||
landing_country = db.relationship("Country", foreign_keys=[landing_country_id], backref=db.backref("logbook_entries_landing", order_by=db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null()).desc()))
|
||||
|
||||
@hybrid_property
|
||||
def duration(self):
|
||||
|
|
@ -38,7 +36,7 @@ class Logbook(db.Model):
|
|||
|
||||
@duration.expression
|
||||
def duration(cls):
|
||||
return case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != null() and cls.takeoff_timestamp != null())
|
||||
return db.case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != db.null() and cls.takeoff_timestamp != db.null())
|
||||
|
||||
@hybrid_property
|
||||
def reference_timestamp(self):
|
||||
|
|
@ -46,13 +44,13 @@ class Logbook(db.Model):
|
|||
|
||||
@reference_timestamp.expression
|
||||
def reference_timestamp(cls):
|
||||
return case({True: cls.takeoff_timestamp, False: cls.landing_timestamp}, cls.takeoff_timestamp != null())
|
||||
return db.case({True: cls.takeoff_timestamp, False: cls.landing_timestamp}, cls.takeoff_timestamp != db.null())
|
||||
|
||||
#__table_args__ = (db.Index('idx_logbook_reference_timestamp', case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null())),)
|
||||
#__table_args__ = (db.Index('idx_logbook_reference_timestamp', db.case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != db.null())),)
|
||||
# FIXME: does not work...
|
||||
|
||||
# FIXME: this does not throw an error as the __table_args__ above, but there is no index created
|
||||
#_wrapped_case = f"({case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != null())})"
|
||||
#_wrapped_case = f"({db.case(whens={True: Logbook.takeoff_timestamp, False: Logbook.landing_timestamp}, value=Logbook.takeoff_timestamp != db.null())})"
|
||||
#Index("idx_logbook_reference_timestamp", _wrapped_case)
|
||||
|
||||
# TODO:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from geoalchemy2.types import Geometry
|
|||
from .geo import Location
|
||||
|
||||
from app import db
|
||||
from sqlalchemy import Index
|
||||
|
||||
from .airport import Airport
|
||||
|
||||
|
|
@ -35,7 +34,7 @@ class Receiver(db.Model):
|
|||
airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True)
|
||||
airport = db.relationship("Airport", foreign_keys=[airport_id], backref=db.backref("receivers", order_by="Receiver.name.asc()"))
|
||||
|
||||
__table_args__ = (Index('idx_receivers_name_uc', 'name', unique=True), )
|
||||
__table_args__ = (db.Index('idx_receivers_name_uc', 'name', unique=True), )
|
||||
|
||||
@property
|
||||
def location(self):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
|
||||
class ReceiverStatistic(db.Model):
|
||||
__tablename__ = "receiver_statistics"
|
||||
|
|
@ -20,6 +17,6 @@ class ReceiverStatistic(db.Model):
|
|||
|
||||
# Relations
|
||||
receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True)
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("statistics", order_by=date.desc()))
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("statistics", order_by=date.desc()))
|
||||
|
||||
__table_args__ = (Index('idx_receiver_statistics_uc', 'date', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
__table_args__ = (db.Index('idx_receiver_statistics_uc', 'date', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
|
||||
class RelationStatistic(db.Model):
|
||||
__tablename__ = "relation_statistics"
|
||||
|
|
@ -18,9 +15,9 @@ class RelationStatistic(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("relation_stats", order_by=date))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("relation_stats", order_by=date))
|
||||
|
||||
receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True)
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("relation_stats", order_by=date))
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("relation_stats", order_by=date))
|
||||
|
||||
__table_args__ = (Index('idx_relation_statistics_uc', 'date', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
__table_args__ = (db.Index('idx_relation_statistics_uc', 'date', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), )
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import datetime
|
||||
|
||||
from sqlalchemy.ext.hybrid import hybrid_property
|
||||
from sqlalchemy import Index
|
||||
|
||||
from app import db
|
||||
from app.model.aircraft_type import AircraftType
|
||||
|
|
@ -22,7 +21,7 @@ class Sender(db.Model):
|
|||
hardware_version = db.Column(db.SmallInteger)
|
||||
real_address = db.Column(db.String(6))
|
||||
|
||||
__table_args__ = (Index('idx_senders_name_uc', 'name', unique=True), )
|
||||
__table_args__ = (db.Index('idx_senders_name_uc', 'name', unique=True), )
|
||||
|
||||
def __repr__(self):
|
||||
return "<Sender: %s,%s,%s,%s,%s,%s>" % (self.address, self.aircraft_type, self.stealth, self.software_version, self.hardware_version, self.real_address)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
from sqlalchemy.dialects.postgresql import JSON
|
||||
|
||||
class SenderDirectionStatistic(db.Model):
|
||||
|
|
@ -16,9 +13,9 @@ class SenderDirectionStatistic(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("direction_stats", order_by=directions_count.desc()))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("direction_stats", order_by=directions_count.desc()))
|
||||
|
||||
receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True)
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("direction_stats", order_by=directions_count.desc()))
|
||||
receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("direction_stats", order_by=directions_count.desc()))
|
||||
|
||||
__table_args__ = (Index('idx_sender_direction_statistics_uc', 'sender_id', 'receiver_id', unique=True), )
|
||||
__table_args__ = (db.Index('idx_sender_direction_statistics_uc', 'sender_id', 'receiver_id', unique=True), )
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from app import db
|
|||
from .sender_info_origin import SenderInfoOrigin
|
||||
from .aircraft_type import AircraftType
|
||||
|
||||
from sqlalchemy.orm import backref
|
||||
#from sqlalchemy.dialects.postgresql import ENUM
|
||||
|
||||
class SenderInfo(db.Model):
|
||||
|
|
@ -22,7 +21,7 @@ class SenderInfo(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("infos", order_by=address_origin))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("infos", order_by=address_origin))
|
||||
|
||||
def __repr__(self):
|
||||
return "<SenderInfo: %s,%s,%s,%s,%s,%s,%s,%s,%s>" % (
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
from .aircraft_type import AircraftType
|
||||
|
||||
from sqlalchemy.dialects.postgresql import ENUM
|
||||
|
|
@ -23,4 +20,4 @@ class SenderPositionStatistic(db.Model):
|
|||
|
||||
messages_count = db.Column(db.Integer)
|
||||
|
||||
__table_args__ = (Index('idx_sender_position_statistics_uc', 'date', 'dstcall', 'address_type', 'aircraft_type', 'stealth', 'software_version', 'hardware_version', unique=True), )
|
||||
__table_args__ = (db.Index('idx_sender_position_statistics_uc', 'date', 'dstcall', 'address_type', 'aircraft_type', 'stealth', 'software_version', 'hardware_version', unique=True), )
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
from app import db
|
||||
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy.orm import backref
|
||||
|
||||
|
||||
class SenderStatistic(db.Model):
|
||||
__tablename__ = "sender_statistics"
|
||||
|
|
@ -20,6 +17,6 @@ class SenderStatistic(db.Model):
|
|||
|
||||
# Relations
|
||||
sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True)
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("statistics", order_by=date.desc()))
|
||||
sender = db.relationship("Sender", foreign_keys=[sender_id], backref=db.backref("statistics", order_by=date.desc()))
|
||||
|
||||
__table_args__ = (Index('idx_sender_statistics_uc', 'date', 'sender_id', 'is_trustworthy', unique=True), )
|
||||
__table_args__ = (db.Index('idx_sender_statistics_uc', 'date', 'sender_id', 'is_trustworthy', unique=True), )
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from app import db
|
||||
from sqlalchemy import Index
|
||||
|
||||
|
||||
class TakeoffLanding(db.Model):
|
||||
|
|
@ -21,4 +20,4 @@ class TakeoffLanding(db.Model):
|
|||
country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="CASCADE"), index=True)
|
||||
country = db.relationship("Country", foreign_keys=[country_id], backref="takeoff_landings")
|
||||
|
||||
__table_args__ = (Index('idx_takeoff_landings_uc', 'timestamp', 'sender_id', 'airport_id', unique=True), )
|
||||
__table_args__ = (db.Index('idx_takeoff_landings_uc', 'timestamp', 'sender_id', 'airport_id', unique=True), )
|
||||
|
|
|
|||
|
|
@ -36,10 +36,10 @@ def update_logbook(offset_days=None):
|
|||
|
||||
|
||||
@celery.task(name="update_logbook_max_altitude")
|
||||
def update_logbook_max_altitude(offset_days=0):
|
||||
def update_logbook_max_altitude():
|
||||
"""Add max altitudes in logbook when flight is complete (takeoff and landing)."""
|
||||
|
||||
result = logbook_update_max_altitudes(offset_days=offset_days)
|
||||
result = logbook_update_max_altitudes()
|
||||
return result
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@
|
|||
<tr>
|
||||
<td>{{ loop.index }}</td>
|
||||
<td><img src="{{ url_for('static', filename='img/Transparent.gif') }}" class="flag flag-{{ receiver.country.iso2|lower }}" alt="{{ receiver.country.iso2 }}"/></td>
|
||||
<td><a href="{{ url_for('main.receiver_detail', receiver_id=receiver.id) }}">{{ receiver.name }}</a></td>
|
||||
<td><a href="{{ url_for('main.airport_detail', airport_id=receiver.airport.id) }}">{{ receiver.airport.name }}</a></td>
|
||||
<td>{{ receiver|to_html_link|safe }}</td>
|
||||
<td>{{ receiver.airport|to_html_link|safe }}</td>
|
||||
<td>{{ receiver.altitude|int }} m</td>
|
||||
<td>{{ receiver.lastseen|timestamp_to_status|safe }}</td>
|
||||
<td>{{ receiver.version if receiver.version else '-' }}</td>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
<tr>
|
||||
<td>{{ loop.index }}</td>
|
||||
<td>{{ entry.sender|to_html_link|safe }}</a></td>
|
||||
<td>{% if entry.sender.infos|length > 0 %}{{ entry.sender.infos[0].aircraft }}{% endif %}</td>
|
||||
<td>{% if entry.sender.infos|length > 0 %}{{ entry.sender.infos[0].aircraft }}{% else %}-{% endif %}</td>
|
||||
<td class="text-right">{{ '%0.1f' | format(entry.max_distance/1000.0) }}</td>
|
||||
<td class="text-right">{{ '%0.1f' | format(entry.max_normalized_quality) }}</td>
|
||||
<td class="text-right">{{ entry.receivers_count }}</td>
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue