kopia lustrzana https://github.com/ogre/habboy
801 wiersze
28 KiB
Python
801 wiersze
28 KiB
Python
#!/usr/bin/env python3
|
|
|
|
|
|
import os
|
|
import string
|
|
import threading
|
|
from pprint import pprint, pformat
|
|
# import bisect
|
|
import datetime
|
|
import dateutil.parser
|
|
import traceback
|
|
import json
|
|
import math
|
|
from copy import deepcopy
|
|
|
|
import formatSentence
|
|
import HabHubInterface
|
|
|
|
import sqlite3
|
|
from sqlite3 import Error as sqerr
|
|
|
|
|
|
C_BLACK = "\033[1;30m"
|
|
C_RED = "\033[1;31m"
|
|
C_GREEN = "\033[1;32m"
|
|
C_BROWN = "\033[1;33m"
|
|
C_BLUE = "\033[1;34m"
|
|
C_MAGENTA = "\033[1;35m"
|
|
C_CYAN = "\033[1;36m"
|
|
C_LIGHTGREY = "\033[1;37m"
|
|
C_OFF = "\033[0m"
|
|
C_CLEAR = "\033[2K"
|
|
|
|
|
|
def sentence_time_formatter(i_str):
|
|
result = datetime.datetime.utcnow()
|
|
in_time = None
|
|
if ":" in i_str:
|
|
in_time = datetime.datetime.strptime(i_str, "%H:%M:%S")
|
|
else:
|
|
if len(i_str) < 6:
|
|
i_str = i_str.zfill(6)
|
|
in_time = datetime.datetime.strptime(i_str, "%H%M%S")
|
|
|
|
result = result.replace(hour = in_time.hour)
|
|
result = result.replace(minute = in_time.minute)
|
|
result = result.replace(second = in_time.second)
|
|
result = result.replace(microsecond = 0)
|
|
return result
|
|
|
|
def coordinate_formatter(i_coord, i_format):
|
|
if i_format == 'dd.dddd': # decimals
|
|
degrees = float(i_coord)
|
|
return degrees
|
|
elif i_format == 'ddmm.mmmm': # degrees/minutes 02145.4512 = 21 degrees, 45 minutes 45.12 seconds
|
|
coord = float(i_coord)
|
|
degrees = math.trunc(coord/100.0)
|
|
minutes = coord - 100.0 * degrees
|
|
return (degrees + minutes/60.0)
|
|
|
|
def sensor_type_converter( i_sensor ):
|
|
s_name = i_sensor['name']
|
|
s_type = i_sensor['sensor']
|
|
if s_type == "base.ascii_int":
|
|
return lambda x: int(float(x))
|
|
elif s_type == "base.ascii_float":
|
|
return lambda x: float(x)
|
|
elif s_type == "base.string":
|
|
if s_name == 'time':
|
|
return sentence_time_formatter
|
|
return lambda x: x
|
|
elif s_type == "stdtelem.time":
|
|
return sentence_time_formatter
|
|
elif s_type == "stdtelem.coordinate":
|
|
if 'format' in i_sensor:
|
|
return lambda x: coordinate_formatter(x, i_sensor['format'])
|
|
else:
|
|
return lambda x: float(x)
|
|
else:
|
|
lambda x: x
|
|
|
|
def is_sensor_numeric(i_sensor_type):
|
|
if i_sensor_type == "base.ascii_int":
|
|
return True
|
|
elif i_sensor_type == "base.ascii_float":
|
|
return True
|
|
elif i_sensor_type == "base.string":
|
|
return False
|
|
elif i_sensor_type == "stdtelem.time":
|
|
return False
|
|
elif i_sensor_type == "stdtelem.coordinate":
|
|
return False
|
|
else:
|
|
return False
|
|
|
|
def sensor_sql_type(i_sensor):
|
|
s_type = i_sensor["sensor"]
|
|
if s_type == "base.ascii_int":
|
|
return "integer"
|
|
elif s_type == "base.ascii_float":
|
|
return "real"
|
|
elif s_type == "base.string":
|
|
if(i_sensor['name'] == 'time'):
|
|
return 'timestamp'
|
|
return "text"
|
|
elif s_type == "stdtelem.time":
|
|
return "timestamp"
|
|
elif s_type == "stdtelem.coordinate":
|
|
return "real"
|
|
else:
|
|
return "text"
|
|
|
|
class VehicleData(object):
|
|
"""
|
|
keeps parsed data for specific callsign
|
|
"""
|
|
|
|
def __init__(self, i_sentence_info):
|
|
# print("VehicleData init")
|
|
# pprint(i_sentence_info)
|
|
|
|
if not i_sentence_info:
|
|
print('VehicleData init, no sentence info')
|
|
raise ValueError
|
|
|
|
self.__IDs = [] # list of sentence TIMESTAMPS
|
|
|
|
# indexed with telemetry sensor name
|
|
# each item is a dict
|
|
# self.__data['altitude']['values'] = [] # raw values
|
|
# self.__data['altitude']['accum'] = [] # sum of values
|
|
# self.__data['altitude']['min'] = []
|
|
# self.__data['altitude']['max'] = []
|
|
# self.__data['altitude']['avg'] = []
|
|
self.__data = {}
|
|
|
|
# info about telemetry datatypes
|
|
# indexed by position in sentence, excluding callsing
|
|
self.__data_info = deepcopy(i_sentence_info)
|
|
# self.__data_info = HabHubInterface.getSentenceInfo(
|
|
# self.__payloadName, i_sentence
|
|
# )
|
|
|
|
if not self.__data_info:
|
|
print(10 * "VehicleData init - no sentence info. exit.\n")
|
|
return
|
|
raise ValueError
|
|
|
|
for i in range(len(self.__data_info)):
|
|
for k in self.__data_info[i]:
|
|
self.__data_info[i][str(k)] = str(self.__data_info[i][k])
|
|
|
|
for i in range(len(self.__data_info)):
|
|
if "sensor" not in self.__data_info[i]:
|
|
continue
|
|
self.__data_info[i]["converter"] = sensor_type_converter( self.__data_info[i] )
|
|
self.__data_info[i]["is_numeric"] = is_sensor_numeric( self.__data_info[i]["sensor"] )
|
|
|
|
def __repr__(self):
|
|
print("Vehicle Data:")
|
|
print("__data")
|
|
pprint(self.__data)
|
|
pprint(self.__data_info)
|
|
|
|
def to_str(self):
|
|
return pformat(self.__data)
|
|
|
|
def add(self, i_sentence, i_sentence_timestamp):
|
|
sentences = formatSentence.formatSentence(i_sentence)
|
|
for s in sentences:
|
|
|
|
sensor_values_str_arr = formatSentence.getData(s)
|
|
if len(sensor_values_str_arr) != len(self.__data_info):
|
|
pprint(self.__data_info)
|
|
raise ValueError(
|
|
"VehicleData ",
|
|
# self.__payloadName,
|
|
" Wrong Sentence Format (values count): ",
|
|
i_sentence,
|
|
len(sensor_values_str_arr),
|
|
len(self.__data_info),
|
|
)
|
|
|
|
sentence_id = i_sentence_timestamp
|
|
|
|
# do not accept sentences older than last - this would require to recompute all stats
|
|
if self.__IDs and sentence_id <= self.__IDs[-1]:
|
|
return
|
|
|
|
self.__IDs.append(sentence_id)
|
|
# insert_index = self.__IDs.index(sentence_id)
|
|
|
|
# sentence_time = sentence_time_formatter(sensor_values_str_arr[1])
|
|
# self.__times.append(i_sentence_timestamp)
|
|
|
|
if not self.__data:
|
|
for i in range( 2, len(sensor_values_str_arr) ): # skip sentence_id and time
|
|
sensor = self.__data_info[i]["name"]
|
|
self.__data[sensor] = {}
|
|
self.__data[sensor]["is_numeric"] = self.__data_info[i]["is_numeric"]
|
|
self.__data[sensor]["values"] = [] # raw values
|
|
if self.__data_info[i]["is_numeric"]:
|
|
self.__data[sensor]["accum"] = [] # sum of values
|
|
self.__data[sensor]["min"] = []
|
|
self.__data[sensor]["max"] = []
|
|
self.__data[sensor]["avg"] = []
|
|
self.__data[sensor]["dVdT"] = []
|
|
|
|
for i in range(2, len(sensor_values_str_arr)): # skip sentence_id and time
|
|
sensor = self.__data_info[i]["name"]
|
|
|
|
# value
|
|
converter = self.__data_info[i]["converter"]
|
|
value = converter(sensor_values_str_arr[i])
|
|
self.__data[sensor]["values"].append(value)
|
|
|
|
if not self.__data_info[i]["is_numeric"]:
|
|
continue
|
|
|
|
# update min/max/avg
|
|
#
|
|
self.__data[sensor]["accum"].append(value)
|
|
self.__data[sensor]["min"].append(value)
|
|
self.__data[sensor]["max"].append(value)
|
|
self.__data[sensor]["avg"].append(value)
|
|
self.__data[sensor]["dVdT"].append(value) # not a good initial value - fix later
|
|
|
|
if len(self.__data[sensor]["values"]) > 1:
|
|
self.__data[sensor]["min"][-1] = min(
|
|
self.__data[sensor]["min"][-2],
|
|
value
|
|
)
|
|
self.__data[sensor]["max"][-1] = max(
|
|
self.__data[sensor]["max"][-2],
|
|
value
|
|
)
|
|
self.__data[sensor]["accum"][-1] = self.__data[sensor]["accum"][-2] + value
|
|
self.__data[sensor]["avg"][-1] = float(self.__data[sensor]["accum"][-1]) / len(self.__data[sensor]["accum"])
|
|
|
|
|
|
# compute change over time - with dT at least 5 seconds
|
|
#
|
|
|
|
prev_index = len(self.__IDs) - 2
|
|
prev_time = self.__IDs[prev_index]
|
|
prev_value = self.__data[sensor]["values"][prev_index]
|
|
dT = float( (i_sentence_timestamp - prev_time).total_seconds() ) or 1
|
|
dVdT = float(value-prev_value) / dT
|
|
|
|
while prev_index >= 0:
|
|
prev_time = self.__IDs[prev_index]
|
|
dT = float( (i_sentence_timestamp - prev_time).total_seconds() )
|
|
if dT < 5:
|
|
prev_index -= 1
|
|
continue
|
|
else:
|
|
prev_value = self.__data[sensor]["values"][prev_index]
|
|
dVdT = float(value-prev_value) / dT
|
|
break
|
|
self.__data[sensor]["dVdT"][-1] = dVdT
|
|
|
|
# fix first dVdT value
|
|
if len(self.__data[sensor]["dVdT"]) == 2:
|
|
self.__data[sensor]["dVdT"][0] = self.__data[sensor]["dVdT"][1]
|
|
|
|
def get_last(self, i_sensor_name):
|
|
result = {}
|
|
try:
|
|
result["is_numeric"] = self.__data[i_sensor_name]["is_numeric"]
|
|
result["values"] = [self.__data[i_sensor_name]["values"][-1]]
|
|
result["times"] = [self.__IDs[-1]]
|
|
if self.__data[i_sensor_name]["is_numeric"]:
|
|
result["min"] = [self.__data[i_sensor_name]["min"][-1]]
|
|
result["max"] = [self.__data[i_sensor_name]["max"][-1]]
|
|
result["avg"] = [self.__data[i_sensor_name]["avg"][-1]]
|
|
result["dVdT"] = [self.__data[i_sensor_name]["dVdT"][-1]]
|
|
except KeyError:
|
|
return result
|
|
return result
|
|
|
|
def get_by_time(self, i_sensor_name, i_time):
|
|
"""
|
|
return sensor data past i_time (excluded)
|
|
"""
|
|
|
|
if i_sensor_name not in self.__data:
|
|
return {}
|
|
|
|
left_i = 0
|
|
for i in range(len(self.__IDs)):
|
|
left_i = i
|
|
# print(self.__IDs[left_i], i_time, self.__IDs[left_i] > i_time)
|
|
if self.__IDs[left_i] > i_time:
|
|
break
|
|
|
|
result = {}
|
|
result["is_numeric"] = self.__data[i_sensor_name]["is_numeric"]
|
|
result["values"] = self.__data[i_sensor_name]["values"][left_i:]
|
|
result["times"] = self.__IDs[left_i:]
|
|
if self.__data[i_sensor_name]["is_numeric"]:
|
|
result["min"] = self.__data[i_sensor_name]["min"][left_i:]
|
|
result["max"] = self.__data[i_sensor_name]["max"][left_i:]
|
|
result["avg"] = self.__data[i_sensor_name]["avg"][left_i:]
|
|
result["dVdT"] = self.__data[i_sensor_name]["dVdT"][left_i:]
|
|
# print(i_sensor_name, len(result['times']),len(result['values']), len(result['dt']) )
|
|
return result
|
|
|
|
def sensorsList(self):
|
|
# if self.__data: return list(self.__data.keys())
|
|
if self.__data_info:
|
|
return [ s['name'] for s in self.__data_info ]
|
|
return [None]
|
|
|
|
def sensorsInfo(self):
|
|
if self.__data_info:
|
|
return copy.deepcopy( self.__data_info )
|
|
# return [ s['name'] for s in self.__data_info ]
|
|
return [None]
|
|
|
|
def verify(self):
|
|
if self.__IDs != sorted(self.__IDs):
|
|
print ("IDs - unsorted")
|
|
if self.__IDs != sorted(self.__IDs):
|
|
print ('times unsorted')
|
|
pprint(self.__IDs)
|
|
|
|
|
|
class SentencesDB(object):
|
|
def __init__(self, db_file):
|
|
if not db_file or db_file == "":
|
|
raise ValueError("SentencesDB: no file specified.")
|
|
|
|
self.__mutex = threading.Lock()
|
|
|
|
self.__sensorsInfo = (
|
|
{}
|
|
) # functions to convert from string to real data, indexed with payloadId
|
|
self.__sqldb = None
|
|
self.__sqldb_file = db_file
|
|
self.__callsignToPayloadId = {}
|
|
self.__sensorToPosition = {} # convert payload_id/sensor_name to position in sentence INCLUDING callsign
|
|
|
|
self.__vehicles_data = {} # class VehicleData, indexed by payloadId
|
|
|
|
self.__initSQLDB()
|
|
|
|
def __initSQLDB(self):
|
|
if self.__sqldb:
|
|
return
|
|
self.__sqldb = sqlite3.connect(self.__sqldb_file, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
|
|
self.__sqldb.row_factory = sqlite3.Row
|
|
self.__createPayloadInfoTable()
|
|
self.__loadSensorsInfo()
|
|
for payload_id in self.__sensorsInfo:
|
|
self.__createPayloadTable(payload_id)
|
|
self.__initVehiclesData()
|
|
|
|
def __createPayloadInfoTable(self):
|
|
"""
|
|
one global metadata table
|
|
keeps info for each table
|
|
"""
|
|
|
|
meta_table_creation_ = "CREATE TABLE IF NOT EXISTS PayloadInfo ("
|
|
meta_table_creation_ += " PayloadId TEXT PRIMARY KEY,"
|
|
meta_table_creation_ += " callsign TEXT NOT NULL,"
|
|
meta_table_creation_ += " SentenceInfo TEXT NOT NULL"
|
|
meta_table_creation_ += ");"
|
|
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute(meta_table_creation_)
|
|
self.__sqldb.commit()
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
except:
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
|
|
def __createPayloadTable(self, i_payloadId):
|
|
"""
|
|
per payload table
|
|
"""
|
|
|
|
payload_table_creation_ = "CREATE TABLE IF NOT EXISTS {} (".format(
|
|
"P_" + i_payloadId
|
|
)
|
|
|
|
for sensor_info in self.__sensorsInfo[i_payloadId]:
|
|
payload_table_creation_ += (
|
|
"\n\t" + sensor_info["name"] + " " + sensor_info["sql_data_type"] + ","
|
|
)
|
|
|
|
payload_table_creation_ += "\n\t_SENTENCE text,\n\t_RECEIVER text,\n\t_INSERT_TIME timestamp,"
|
|
payload_table_creation_ += "\n\tPRIMARY KEY(sentence_id, time, _RECEIVER)\n);"
|
|
|
|
cur = self.__sqldb.cursor()
|
|
try:
|
|
cur.execute(payload_table_creation_)
|
|
self.__sqldb.commit()
|
|
except sqerr as e:
|
|
print(
|
|
"error cur.execute(payload_table_creation_) - payload_id = ",
|
|
i_payloadId,
|
|
)
|
|
print(e)
|
|
print(payload_table_creation_)
|
|
import traceback
|
|
print(traceback.format_exc())
|
|
except:
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
|
|
def __loadSensorsInfo(self):
|
|
if self.__sensorsInfo and len(self.__sensorsInfo.keys()):
|
|
print("__loadSensorsInfo - early exit")
|
|
return
|
|
|
|
# load from disk
|
|
#
|
|
|
|
habitat_data = {}
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute("SELECT PayloadId,callsign,SentenceInfo FROM PayloadInfo;")
|
|
for PayloadId, callsign, sensors_info_str in cur.fetchall():
|
|
habitat_data[PayloadId] = json.loads(sensors_info_str)
|
|
self.__callsignToPayloadId[callsign] = PayloadId
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
|
|
# convert from habitat data
|
|
#
|
|
|
|
for PayloadId in habitat_data:
|
|
|
|
self.__sensorsInfo[PayloadId] = []
|
|
|
|
self.__sensorToPosition[PayloadId] = {'callsign': 0}
|
|
sensorPostiion = 1 # start with 1 because callsign is counted in indexing
|
|
|
|
for sensor_info in habitat_data[PayloadId]:
|
|
sensor_info["sql_data_type"] = sensor_sql_type( sensor_info )
|
|
sensor_info["converter"] = sensor_type_converter( sensor_info )
|
|
# sensor_info["sql_primary_key"] = sensor_info["name"] == "sentence_id"
|
|
self.__sensorsInfo[PayloadId].append(sensor_info)
|
|
self.__sensorToPosition[PayloadId][sensor_info['name']] = sensorPostiion
|
|
sensorPostiion += 1
|
|
|
|
|
|
def __initVehiclesData(self):
|
|
# return
|
|
if not self.__sqldb:
|
|
return
|
|
|
|
allPayloadIds = self.getAllPayloadIds()
|
|
print("allPayloadIds:")
|
|
pprint(allPayloadIds)
|
|
sentences_count = {}
|
|
|
|
cur = self.__sqldb.cursor()
|
|
for pid,callsign in allPayloadIds:
|
|
sentences_count[pid] = 0
|
|
|
|
# update vehicles data
|
|
if pid not in self.__vehicles_data:
|
|
_t = VehicleData(self.getSensorsInfo(pid))
|
|
self.__vehicles_data[pid] = _t
|
|
|
|
try:
|
|
cur.execute(" select time,_SENTENCE from {};".format("P_" + pid))
|
|
except sqlite3.OperationalError as e:
|
|
if 'no such table: P_' in str(e):
|
|
continue # no data/table for this payload_id
|
|
else:
|
|
print(traceback.format_exc())
|
|
continue
|
|
|
|
for time, sentence in cur.fetchall():
|
|
try:
|
|
self.__vehicles_data[pid].add(sentence, time)
|
|
sentences_count[pid] += 1
|
|
except:
|
|
print("Can't parse ", sentence)
|
|
print(traceback.format_exc())
|
|
|
|
# if pid in self.__vehicles_data: print( self.__vehicles_data[pid].to_str() )
|
|
|
|
|
|
if sentences_count:
|
|
print("Restored Vehicles Data from DB:")
|
|
for k, v in sentences_count.items():
|
|
callsign = self.payloadIdToCallsign(k)
|
|
print("\t", callsign, v)
|
|
|
|
# for pid in self.__vehicles_data:
|
|
# print(pid)
|
|
# print(self.__vehicles_data[pid])
|
|
|
|
# for pid in self.__vehicles_data:
|
|
# self.__vehicles_data[pid].verify()
|
|
|
|
def callsignToPayloadId(self, i_callsign):
|
|
if i_callsign in self.__callsignToPayloadId:
|
|
return self.__callsignToPayloadId[i_callsign]
|
|
return None
|
|
|
|
def create_empty(self):
|
|
self.__initSQLDB()
|
|
|
|
|
|
def payloadIdToCallsign(self, i_payloadId):
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute(
|
|
'SELECT callsign FROM PayloadInfo WHERE PayloadId is "{}";'.format(
|
|
str(i_payloadId)
|
|
)
|
|
)
|
|
return cur.fetchall()[0][0]
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
print(traceback.format_exc())
|
|
return None
|
|
except IndexError:
|
|
return None
|
|
|
|
def updatePayloadInfo(self, i_payloadId, i_sentence=None):
|
|
callsign = HabHubInterface.getCallsignsForPayloadId(i_payloadId)
|
|
|
|
sentence_info = HabHubInterface.getSentenceInfo(i_payloadId, i_sentence)
|
|
if not sentence_info:
|
|
print("UpdatePayloadInfo - No Sentence Info for payload ", i_payloadId)
|
|
return
|
|
|
|
# HACK !
|
|
# time field can have many names, lets always use 'time'
|
|
for i in range(len(sentence_info)):
|
|
if 'time' in sentence_info[i]['name']:
|
|
sentence_info[i]['name'] = 'time'
|
|
break
|
|
|
|
|
|
sql_insert = (
|
|
"REPLACE INTO PayloadInfo(PayloadId, callsign, SentenceInfo) VALUES(?,?,?)"
|
|
)
|
|
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute(
|
|
sql_insert,
|
|
(
|
|
i_payloadId,
|
|
callsign,
|
|
json.dumps(sentence_info, sort_keys=True, indent=4),
|
|
),
|
|
)
|
|
self.__sqldb.commit()
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
|
|
def getAllPayloadIds(self):
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute("select PayloadId, callsign from PayloadInfo;")
|
|
res = [ [x[0], x[1]] for x in cur.fetchall()]
|
|
return res
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
|
|
print(traceback.format_exc())
|
|
|
|
def getInfo(self):
|
|
try:
|
|
with self.__mutex:
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute("select * from PayloadInfo;")
|
|
res = [list(x) for x in cur.fetchall()]
|
|
res = [ [x[0], x[1], json.loads(x[2])] for x in res]
|
|
return res
|
|
except sqerr as e:
|
|
print(e)
|
|
import traceback
|
|
print(traceback.format_exc())
|
|
|
|
def insert(self, i_sentence, i_RECEIVER="unknown"):
|
|
if not i_sentence:
|
|
return
|
|
|
|
sentences = formatSentence.formatSentence(i_sentence)
|
|
if not sentences:
|
|
return
|
|
|
|
with self.__mutex:
|
|
for sentence in sentences:
|
|
callsign = formatSentence.sentenceToCallsign(sentence)
|
|
payload_id = self.callsignToPayloadId(callsign)
|
|
if not payload_id:
|
|
print("DB Insert -- Unknown callsign: ", callsign)
|
|
continue
|
|
|
|
sent_id = formatSentence.sentenceToId(sentence)
|
|
|
|
if not self.__sensorsInfo:
|
|
print("DB Insert -- No sentence data info. returning.")
|
|
return
|
|
|
|
# get data from sentence and convert from string to real types
|
|
sensors_data = formatSentence.getData(sentence)
|
|
try:
|
|
for i in range(len(sensors_data)):
|
|
converter = self.__sensorsInfo[payload_id][i]["converter"]
|
|
sensors_data[i] = converter(sensors_data[i])
|
|
except:
|
|
print("DB Insert -- Error converting sentence fields to data.")
|
|
print(traceback.format_exc())
|
|
continue
|
|
|
|
# _SENTENCE field
|
|
sensors_data.append(sentence)
|
|
|
|
# _RECEIVER field
|
|
sensors_data.append(str(i_RECEIVER))
|
|
|
|
# _INSERT_TIME field
|
|
sensors_data.append(datetime.datetime.utcnow())
|
|
|
|
|
|
sql_insert = "INSERT INTO {}(".format("P_" + payload_id)
|
|
sql_insert += ",".join(
|
|
[sensor["name"] for sensor in self.__sensorsInfo[payload_id]]
|
|
)
|
|
sql_insert += ", _SENTENCE, _RECEIVER, _INSERT_TIME"
|
|
sql_insert += ")"
|
|
sql_insert += (
|
|
"\n\tVALUES(" + ",".join(["?"] * len(sensors_data)) + ")"
|
|
)
|
|
|
|
cur = self.__sqldb.cursor()
|
|
try:
|
|
cur.execute(sql_insert, sensors_data)
|
|
self.__sqldb.commit()
|
|
# print( cur.lastrowid )
|
|
except sqlite3.IntegrityError:
|
|
pass
|
|
# print("sqlite3.IntegrityError - already inserted ?")
|
|
except sqerr as e:
|
|
print(e)
|
|
|
|
# update vehicles data
|
|
if payload_id not in self.__vehicles_data:
|
|
self.__vehicles_data[payload_id] = VehicleData(
|
|
self.getSensorsInfo(payload_id)
|
|
)
|
|
try:
|
|
sentence_timestamp = sensors_data[ self.__sensorToPosition[payload_id]['time'] -1 ]
|
|
self.__vehicles_data[payload_id].add(sentence, sentence_timestamp)
|
|
except:
|
|
print("DB Insert -- Can't parse ", sentence)
|
|
print(traceback.format_exc())
|
|
return True
|
|
|
|
def getSensorsList(self, i_payloadId):
|
|
with self.__mutex:
|
|
if i_payloadId in self.__vehicles_data:
|
|
return deepcopy( self.__vehicles_data[i_payloadId].sensorsList() )
|
|
else:
|
|
pass
|
|
# print(self.__vehicles_data.keys())
|
|
return []
|
|
|
|
def getSensorsInfo(self, i_payloadId):
|
|
with self.__mutex:
|
|
if i_payloadId in self.__sensorsInfo:
|
|
return deepcopy( self.__sensorsInfo[i_payloadId] )
|
|
return None
|
|
|
|
def getTelemetryLast(self, i_payloadId, i_sensor_name):
|
|
with self.__mutex:
|
|
if not list(self.__vehicles_data.keys()):
|
|
return {}
|
|
|
|
if i_payloadId not in self.__vehicles_data:
|
|
return {}
|
|
|
|
return self.__vehicles_data[i_payloadId].get_last(i_sensor_name)
|
|
|
|
def getTelemetryByTime(self, i_payloadId, i_sensor_name, i_time):
|
|
_t = i_time
|
|
if type(_t) == type(""):
|
|
# _t = datetime.datetime.strptime(_t, "%Y-%m-%d %H:%M:%S")
|
|
_t = dateutil.parser.parse(_t, ignoretz=1)
|
|
|
|
with self.__mutex:
|
|
if not list(self.__vehicles_data.keys()):
|
|
return []
|
|
|
|
if i_payloadId not in self.__vehicles_data:
|
|
return []
|
|
|
|
return self.__vehicles_data[i_payloadId].get_by_time(i_sensor_name, _t)
|
|
|
|
def getLastSentence(self, i_payloadId):
|
|
if not i_payloadId:
|
|
print("getLastSentence: i_payloadId = ", i_payloadId)
|
|
return
|
|
|
|
if not self.__sqldb:
|
|
return None
|
|
|
|
with self.__mutex:
|
|
tab_name = "P_" + i_payloadId
|
|
cur = self.__sqldb.cursor()
|
|
# cur.execute("select MAX(time) from {};".format(tab_name))
|
|
cur.execute("select MAX(_INSERT_TIME) from {};".format(tab_name))
|
|
last_time = cur.fetchone()[0]
|
|
if last_time:
|
|
cur.execute(
|
|
# "select * from {} where time is '{}';".format(
|
|
"select * from {} where _INSERT_TIME is '{}';".format(
|
|
tab_name, last_time
|
|
)
|
|
)
|
|
values = cur.fetchall()
|
|
return dict(values[0])
|
|
|
|
def getLastSentenceId(self, i_payloadId):
|
|
if not i_payloadId:
|
|
print("getLastSentenceId: i_payloadId = ", i_payloadId)
|
|
return
|
|
|
|
if not self.__sqldb:
|
|
return
|
|
|
|
with self.__mutex:
|
|
tab_name = "P_" + i_payloadId
|
|
cur = self.__sqldb.cursor()
|
|
cur.execute("select MAX(sentence_id) from {};".format(tab_name))
|
|
last_sentence_id = cur.fetchone()[0]
|
|
if last_sentence_id:
|
|
return last_sentence_id
|
|
|
|
return
|
|
|
|
def getReceiversStats(self, i_payloadId):
|
|
if not i_payloadId:
|
|
return
|
|
|
|
if not self.__sqldb:
|
|
return
|
|
|
|
with self.__mutex:
|
|
try:
|
|
c = self.__sqldb.cursor()
|
|
tab_name = "P_" + i_payloadId
|
|
c.execute("SELECT DISTINCT _RECEIVER FROM {}".format(tab_name))
|
|
all_receivers = c.fetchall()
|
|
all_receivers = [a[0] for a in all_receivers]
|
|
# print(all_receivers)
|
|
|
|
receiver_stats = {}
|
|
for r in all_receivers:
|
|
c.execute(
|
|
'SELECT COUNT(sentence_id) FROM "{}" WHERE _RECEIVER = "{}"'.format(
|
|
tab_name, r
|
|
)
|
|
)
|
|
receiver_stats[r] = int(c.fetchone()[0])
|
|
|
|
# pprint(receiver_stats)
|
|
return receiver_stats
|
|
except sqlite3.OperationalError as e:
|
|
if 'no such table: P_' in str(e):
|
|
return None # no data/table for this payload_id
|
|
else:
|
|
print(traceback.format_exc())
|
|
return None
|
|
|
|
return
|
|
|
|
if __name__ == "__main__":
|
|
print(sentence_time_formatter('123456'))
|
|
print(sentence_time_formatter('12:34:56')) |