mirror
/
ogn-python
kopia lustrzana https://github.com/glidernet/ogn-python
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #41 from Meisterschueler/fix/addreceiver 

Bugfix: If an existing receiver was updated, a new receiver was not added
pull/42/head
Fabian P. Schmidt 2016-02-15 20:17:06 +01:00
rodzic ad362fcc1e 9b4a0124e8
commit 4b3302b79e
1 zmienionych plików z 43 dodań i 36 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

79
ogn/collect/receiver.py
Wyświetl plik

@ -14,58 +14,54 @@ logger = get_task_logger(__name__)
@app.task @app.task
def update_receivers(): def update_receivers():
"""Update the receiver table.""" """Update the receiver table."""
# get current receiver data # get the timestamp of last update
last_entry_sq = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry')) \ last_update_query = app.session.query(coalesce(func.max(Receiver.lastseen), '2015-01-01 00:00:00').label('last_entry'))
.subquery() last_update = last_update_query.one().last_entry
# get last receiver beacons since last update
last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name, last_receiver_beacon_sq = app.session.query(ReceiverBeacon.name,
func.min(ReceiverBeacon.timestamp).label('firstseen'),
func.max(ReceiverBeacon.timestamp).label('lastseen')) \ func.max(ReceiverBeacon.timestamp).label('lastseen')) \
.filter(ReceiverBeacon.timestamp >= last_entry_sq.columns.last_entry) \ .filter(ReceiverBeacon.timestamp >= last_update) \
.group_by(ReceiverBeacon.name) \ .group_by(ReceiverBeacon.name) \
.subquery() .subquery()
# update existing receivers receivers_to_update = app.session.query(ReceiverBeacon.name,
sq = app.session.query(ReceiverBeacon.name, ReceiverBeacon.latitude,
ReceiverBeacon.latitude, ReceiverBeacon.longitude,
ReceiverBeacon.longitude, ReceiverBeacon.altitude,
ReceiverBeacon.altitude, last_receiver_beacon_sq.columns.lastseen,
last_receiver_beacon_sq.columns.firstseen, ReceiverBeacon.version,
last_receiver_beacon_sq.columns.lastseen, ReceiverBeacon.platform) \
ReceiverBeacon.version, .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
ReceiverBeacon.platform) \ ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, .subquery()
ReceiverBeacon.timestamp == last_receiver_beacon_sq.columns.lastseen)) \
.subquery()
# set country code to None if lat or lon changed # set country code to None if lat or lon changed
upd = app.session.query(Receiver) \ count = app.session.query(Receiver) \
.filter(and_(Receiver.name == sq.columns.name, .filter(and_(Receiver.name == receivers_to_update.columns.name,
or_(Receiver.latitude != sq.columns.latitude, or_(Receiver.latitude != receivers_to_update.columns.latitude,
Receiver.longitude != sq.columns.longitude))) \ Receiver.longitude != receivers_to_update.columns.longitude))) \
.update({"latitude": sq.columns.latitude, .update({"latitude": receivers_to_update.columns.latitude,
"longitude": sq.columns.longitude, "longitude": receivers_to_update.columns.longitude,
"country_code": null()}) "country_code": null()})
logger.info("Count of receivers who changed lat or lon: {}".format(upd)) logger.info("Count of receivers who changed lat or lon: {}".format(count))
app.session.commit()
# update lastseen of known receivers # update lastseen of known receivers
upd = app.session.query(Receiver) \ count = app.session.query(Receiver) \
.filter(Receiver.name == sq.columns.name) \ .filter(Receiver.name == receivers_to_update.columns.name) \
.update({"altitude": sq.columns.altitude, .update({"altitude": receivers_to_update.columns.altitude,
"lastseen": sq.columns.lastseen, "lastseen": receivers_to_update.columns.lastseen,
"version": sq.columns.version, "version": receivers_to_update.columns.version,
"platform": sq.columns.platform}) "platform": receivers_to_update.columns.platform})
logger.info("Count of receivers who where updated: {}".format(upd)) logger.info("Count of receivers who where updated: {}".format(count))
# add new receivers # add new receivers
empty_sq = app.session.query(ReceiverBeacon.name, empty_sq = app.session.query(ReceiverBeacon.name,
ReceiverBeacon.latitude, ReceiverBeacon.latitude,
ReceiverBeacon.longitude, ReceiverBeacon.longitude,
ReceiverBeacon.altitude, ReceiverBeacon.altitude,
last_receiver_beacon_sq.columns.firstseen,
last_receiver_beacon_sq.columns.lastseen, last_receiver_beacon_sq.columns.lastseen,
ReceiverBeacon.version, ReceiverBeacon.platform) \ ReceiverBeacon.version, ReceiverBeacon.platform) \
.filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name, .filter(and_(ReceiverBeacon.name == last_receiver_beacon_sq.columns.name,
@ -80,7 +76,7 @@ def update_receivers():
receiver.latitude = receiver_beacon.latitude receiver.latitude = receiver_beacon.latitude
receiver.longitude = receiver_beacon.longitude receiver.longitude = receiver_beacon.longitude
receiver.altitude = receiver_beacon.altitude receiver.altitude = receiver_beacon.altitude
receiver.firstseen = receiver_beacon.firstseen receiver.firstseen = None
receiver.lastseen = receiver_beacon.lastseen receiver.lastseen = receiver_beacon.lastseen
receiver.version = receiver_beacon.version receiver.version = receiver_beacon.version
receiver.platform = receiver_beacon.platform receiver.platform = receiver_beacon.platform
@ -88,7 +84,18 @@ def update_receivers():
app.session.add(receiver) app.session.add(receiver)
logger.info("{} added".format(receiver.name)) logger.info("{} added".format(receiver.name))
app.session.commit() # update firstseen if None
firstseen_null_query = app.session.query(Receiver.name,
func.min(ReceiverBeacon.timestamp).label('firstseen')) \
.filter(Receiver.firstseen == null()) \
.join(ReceiverBeacon, Receiver.name == ReceiverBeacon.name) \
.group_by(Receiver.name) \
.subquery()
count = app.session.query(Receiver) \
.filter(Receiver.name == firstseen_null_query.columns.name) \
.update({'firstseen': firstseen_null_query.columns.firstseen})
logger.info("Total: {} receivers added".format(count))
# update country code if None # update country code if None
unknown_country_query = app.session.query(Receiver) \ unknown_country_query = app.session.query(Receiver) \