2016-03-23 16:31:11 +00:00
|
|
|
/*
|
|
|
|
Copyright (c) 2015-2016 Christopher Young
|
|
|
|
Distributable under the terms of The "BSD New"" License
|
|
|
|
that can be found in the LICENSE file, herein included
|
|
|
|
as part of this header.
|
|
|
|
|
|
|
|
datalog.go: Log stratux data as it is received. Bucket data into timestamp time slots.
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2016-03-24 04:23:12 +00:00
|
|
|
"database/sql"
|
2016-03-23 16:31:11 +00:00
|
|
|
"fmt"
|
2016-03-24 04:23:12 +00:00
|
|
|
_ "github.com/mattn/go-sqlite3"
|
2016-03-24 13:33:11 +00:00
|
|
|
"log"
|
|
|
|
"os"
|
2016-03-23 16:31:11 +00:00
|
|
|
"reflect"
|
2016-03-24 03:08:00 +00:00
|
|
|
"strconv"
|
2016-03-23 16:31:11 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2016-03-24 13:33:11 +00:00
|
|
|
LOG_TIMESTAMP_RESOLUTION = 250 * time.Millisecond
|
2016-03-23 16:31:11 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type StratuxTimestamp struct {
|
|
|
|
id int64
|
2016-03-24 05:14:48 +00:00
|
|
|
Time_type_preference int // 0 = stratuxClock, 1 = gpsClock, 2 = gpsClock extrapolated via stratuxClock.
|
|
|
|
StratuxClock_value time.Time
|
|
|
|
GPSClock_value time.Time
|
|
|
|
PreferredTime_value time.Time
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var dataLogTimestamp StratuxTimestamp // Current timestamp bucket.
|
|
|
|
|
|
|
|
/*
|
|
|
|
checkTimestamp().
|
|
|
|
Verify that our current timestamp is within the LOG_TIMESTAMP_RESOLUTION bucket.
|
|
|
|
Returns false if the timestamp was changed, true if it is still valid.
|
|
|
|
*/
|
2016-03-24 04:23:12 +00:00
|
|
|
|
|
|
|
//FIXME: time -> stratuxClock
|
2016-03-23 16:31:11 +00:00
|
|
|
func checkTimestamp() bool {
|
2016-03-24 05:14:48 +00:00
|
|
|
if time.Since(dataLogTimestamp.StratuxClock_value) >= LOG_TIMESTAMP_RESOLUTION {
|
2016-03-23 16:31:11 +00:00
|
|
|
//FIXME: mutex.
|
|
|
|
dataLogTimestamp.id = 0
|
2016-03-24 13:33:11 +00:00
|
|
|
dataLogTimestamp.Time_type_preference = 0 // stratuxClock.
|
|
|
|
dataLogTimestamp.StratuxClock_value = stratuxClock.Time
|
|
|
|
dataLogTimestamp.GPSClock_value = time.Time{}
|
|
|
|
dataLogTimestamp.PreferredTime_value = stratuxClock.Time
|
2016-03-23 16:31:11 +00:00
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
2016-03-24 04:23:12 +00:00
|
|
|
|
2016-03-23 16:31:11 +00:00
|
|
|
type SQLiteMarshal struct {
|
|
|
|
FieldType string
|
2016-03-24 03:08:00 +00:00
|
|
|
Marshal func(v reflect.Value) string
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 03:08:00 +00:00
|
|
|
func boolMarshal(v reflect.Value) string {
|
|
|
|
b := v.Bool()
|
|
|
|
if b {
|
|
|
|
return "1"
|
|
|
|
}
|
|
|
|
return "0"
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 03:08:00 +00:00
|
|
|
func intMarshal(v reflect.Value) string {
|
|
|
|
return strconv.FormatInt(v.Int(), 10)
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 03:08:00 +00:00
|
|
|
func uintMarshal(v reflect.Value) string {
|
|
|
|
return strconv.FormatUint(v.Uint(), 10)
|
|
|
|
}
|
|
|
|
|
|
|
|
func floatMarshal(v reflect.Value) string {
|
|
|
|
return strconv.FormatFloat(v.Float(), 'f', 10, 64)
|
|
|
|
}
|
|
|
|
|
|
|
|
func stringMarshal(v reflect.Value) string {
|
|
|
|
return v.String()
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 03:08:00 +00:00
|
|
|
func notsupportedMarshal(v reflect.Value) string {
|
2016-03-23 16:31:11 +00:00
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
func structCanBeMarshalled(v reflect.Value) bool {
|
|
|
|
m := v.MethodByName("String")
|
|
|
|
if m.IsValid() && !m.IsNil() {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-03-24 03:08:00 +00:00
|
|
|
func structMarshal(v reflect.Value) string {
|
|
|
|
if structCanBeMarshalled(v) {
|
|
|
|
m := v.MethodByName("String")
|
|
|
|
in := make([]reflect.Value, 0)
|
|
|
|
ret := m.Call(in)
|
|
|
|
if len(ret) > 0 {
|
|
|
|
return ret[0].String()
|
|
|
|
}
|
|
|
|
}
|
2016-03-23 16:31:11 +00:00
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
var sqliteMarshalFunctions = map[string]SQLiteMarshal{
|
|
|
|
"bool": {FieldType: "INTEGER", Marshal: boolMarshal},
|
|
|
|
"int": {FieldType: "INTEGER", Marshal: intMarshal},
|
|
|
|
"uint": {FieldType: "INTEGER", Marshal: uintMarshal},
|
|
|
|
"float": {FieldType: "REAL", Marshal: floatMarshal},
|
|
|
|
"string": {FieldType: "TEXT", Marshal: stringMarshal},
|
2016-03-24 03:08:00 +00:00
|
|
|
"struct": {FieldType: "STRING", Marshal: structMarshal},
|
2016-03-23 16:31:11 +00:00
|
|
|
"notsupported": {FieldType: "notsupported", Marshal: notsupportedMarshal},
|
|
|
|
}
|
|
|
|
|
|
|
|
var sqlTypeMap = map[reflect.Kind]string{
|
|
|
|
reflect.Bool: "bool",
|
|
|
|
reflect.Int: "int",
|
|
|
|
reflect.Int8: "int",
|
|
|
|
reflect.Int16: "int",
|
|
|
|
reflect.Int32: "int",
|
|
|
|
reflect.Int64: "int",
|
|
|
|
reflect.Uint: "uint",
|
|
|
|
reflect.Uint8: "uint",
|
|
|
|
reflect.Uint16: "uint",
|
|
|
|
reflect.Uint32: "uint",
|
|
|
|
reflect.Uint64: "uint",
|
|
|
|
reflect.Uintptr: "notsupported",
|
|
|
|
reflect.Float32: "float",
|
|
|
|
reflect.Float64: "float",
|
|
|
|
reflect.Complex64: "notsupported",
|
|
|
|
reflect.Complex128: "notsupported",
|
|
|
|
reflect.Array: "notsupported",
|
|
|
|
reflect.Chan: "notsupported",
|
|
|
|
reflect.Func: "notsupported",
|
|
|
|
reflect.Interface: "notsupported",
|
|
|
|
reflect.Map: "notsupported",
|
|
|
|
reflect.Ptr: "notsupported",
|
|
|
|
reflect.Slice: "notsupported",
|
|
|
|
reflect.String: "string",
|
2016-03-24 03:08:00 +00:00
|
|
|
reflect.Struct: "struct",
|
2016-03-23 16:31:11 +00:00
|
|
|
reflect.UnsafePointer: "notsupported",
|
|
|
|
}
|
|
|
|
|
2016-03-24 04:23:12 +00:00
|
|
|
func makeTable(i interface{}, tbl string, db *sql.DB) {
|
2016-03-23 16:31:11 +00:00
|
|
|
val := reflect.ValueOf(i)
|
|
|
|
|
|
|
|
fields := make([]string, 0)
|
|
|
|
for i := 0; i < val.NumField(); i++ {
|
|
|
|
kind := val.Field(i).Kind()
|
|
|
|
fieldName := val.Type().Field(i).Name
|
|
|
|
sqlTypeAlias := sqlTypeMap[kind]
|
2016-03-24 03:08:00 +00:00
|
|
|
|
|
|
|
// Check that if the field is a struct that it can be marshalled.
|
|
|
|
if sqlTypeAlias == "struct" && !structCanBeMarshalled(val.Field(i)) {
|
|
|
|
continue
|
|
|
|
}
|
2016-03-23 16:31:11 +00:00
|
|
|
if sqlTypeAlias == "notsupported" || fieldName == "id" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
sqlType := sqliteMarshalFunctions[sqlTypeAlias].FieldType
|
|
|
|
s := fieldName + " " + sqlType
|
|
|
|
fields = append(fields, s)
|
|
|
|
}
|
|
|
|
|
2016-03-24 05:14:48 +00:00
|
|
|
// Add the timestamp_id field to link up with the timestamp table.
|
|
|
|
if tbl != "timestamp" {
|
|
|
|
fields = append(fields, "timestamp_id INTEGER")
|
|
|
|
}
|
|
|
|
|
|
|
|
tblCreate := fmt.Sprintf("CREATE TABLE %s (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, %s)", tbl, strings.Join(fields, ", "))
|
|
|
|
_, err := db.Exec(tblCreate)
|
|
|
|
fmt.Printf("%s\n", tblCreate)
|
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("ERROR: %s\n", err.Error())
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-24 05:14:48 +00:00
|
|
|
func insertData(i interface{}, tbl string, db *sql.DB) int64 {
|
2016-03-24 04:23:12 +00:00
|
|
|
checkTimestamp()
|
2016-03-24 03:08:00 +00:00
|
|
|
val := reflect.ValueOf(i)
|
|
|
|
|
2016-03-24 04:23:12 +00:00
|
|
|
keys := make([]string, 0)
|
|
|
|
values := make([]string, 0)
|
2016-03-24 03:08:00 +00:00
|
|
|
for i := 0; i < val.NumField(); i++ {
|
|
|
|
kind := val.Field(i).Kind()
|
|
|
|
fieldName := val.Type().Field(i).Name
|
|
|
|
sqlTypeAlias := sqlTypeMap[kind]
|
|
|
|
|
|
|
|
if sqlTypeAlias == "notsupported" || fieldName == "id" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
v := sqliteMarshalFunctions[sqlTypeAlias].Marshal(val.Field(i))
|
|
|
|
|
2016-03-24 04:23:12 +00:00
|
|
|
keys = append(keys, fieldName)
|
|
|
|
values = append(values, v)
|
2016-03-24 03:08:00 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 05:14:48 +00:00
|
|
|
// Add the timestamp_id field to link up with the timestamp table.
|
|
|
|
if tbl != "timestamp" {
|
|
|
|
keys = append(keys, "timestamp_id")
|
|
|
|
values = append(values, strconv.FormatInt(dataLogTimestamp.id, 10))
|
|
|
|
}
|
|
|
|
|
|
|
|
tblInsert := fmt.Sprintf("INSERT INTO %s (%s) VALUES(%s)", tbl, strings.Join(keys, ","),
|
|
|
|
strings.Join(strings.Split(strings.Repeat("?", len(keys)), ""), ","))
|
|
|
|
|
|
|
|
fmt.Printf("%s\n", tblInsert)
|
|
|
|
ifs := make([]interface{}, len(values))
|
|
|
|
for i := 0; i < len(values); i++ {
|
|
|
|
ifs[i] = values[i]
|
|
|
|
}
|
|
|
|
res, err := db.Exec(tblInsert, ifs...)
|
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("ERROR: %s\n", err.Error())
|
|
|
|
}
|
|
|
|
id, err := res.LastInsertId()
|
|
|
|
if err == nil {
|
|
|
|
return id
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
type DataLogRow struct {
|
|
|
|
tbl string
|
|
|
|
data interface{}
|
|
|
|
}
|
|
|
|
|
|
|
|
var dataLogChan chan DataLogRow
|
|
|
|
|
|
|
|
func dataLogWriter() {
|
|
|
|
dataLogChan := make(chan DataLogRow, 10240)
|
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
// Check if we need to create a new database.
|
|
|
|
createDatabase := false
|
|
|
|
|
|
|
|
if _, err := os.Stat(dataLogFile); os.IsNotExist(err) {
|
|
|
|
createDatabase = true
|
|
|
|
log.Printf("creating new database '%s'.\n", dataLogFile)
|
|
|
|
}
|
|
|
|
|
|
|
|
db, err := sql.Open("sqlite3", dataLogFile)
|
2016-03-24 05:14:48 +00:00
|
|
|
if err != nil {
|
2016-03-24 13:33:11 +00:00
|
|
|
log.Printf("sql.Open(): %s\n", err.Error())
|
2016-03-24 05:14:48 +00:00
|
|
|
}
|
|
|
|
defer db.Close()
|
2016-03-24 04:23:12 +00:00
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
// Do we need to create the database?
|
|
|
|
if createDatabase {
|
|
|
|
makeTable(dataLogTimestamp, "timestamp", db)
|
|
|
|
makeTable(mySituation, "mySituation", db)
|
|
|
|
makeTable(globalStatus, "status", db)
|
|
|
|
makeTable(globalSettings, "settings", db)
|
|
|
|
makeTable(TrafficInfo{}, "traffic", db)
|
|
|
|
}
|
|
|
|
|
2016-03-24 05:14:48 +00:00
|
|
|
for {
|
|
|
|
//FIXME: measure latency from here to end of block. Messages may need to be timestamped *before* executing everything here.
|
|
|
|
r := <-dataLogChan
|
2016-03-24 13:33:11 +00:00
|
|
|
if r.tbl == "mySituation" && isGPSClockValid() {
|
|
|
|
// Piggyback a GPS time update from this update.
|
|
|
|
if t, ok := r.data.(SituationData); ok {
|
|
|
|
dataLogTimestamp.id = 0
|
|
|
|
dataLogTimestamp.Time_type_preference = 1 // gpsClock.
|
|
|
|
dataLogTimestamp.StratuxClock_value = stratuxClock.Time
|
|
|
|
dataLogTimestamp.GPSClock_value = t.GPSTime
|
|
|
|
dataLogTimestamp.PreferredTime_value = t.GPSTime
|
|
|
|
}
|
2016-03-24 04:23:12 +00:00
|
|
|
}
|
2016-03-24 05:14:48 +00:00
|
|
|
|
|
|
|
// Check if our time bucket has expired or has never been entered.
|
|
|
|
if !checkTimestamp() || dataLogTimestamp.id == 0 {
|
|
|
|
dataLogTimestamp.id = insertData(dataLogTimestamp, "timestamp", db)
|
2016-03-24 04:23:12 +00:00
|
|
|
}
|
2016-03-24 05:14:48 +00:00
|
|
|
insertData(r.data, r.tbl, db)
|
2016-03-23 16:31:11 +00:00
|
|
|
}
|
|
|
|
}
|
2016-03-24 03:08:00 +00:00
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
func logSituation() {
|
|
|
|
dataLogChan <- DataLogRow{tbl: "mySituation", data: mySituation}
|
2016-03-24 04:23:12 +00:00
|
|
|
}
|
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
func logStatus() {
|
|
|
|
dataLogChan <- DataLogRow{tbl: "status", data: globalStatus}
|
|
|
|
}
|
|
|
|
|
|
|
|
func logSettings() {
|
|
|
|
dataLogChan <- DataLogRow{tbl: "settings", data: globalSettings}
|
|
|
|
}
|
|
|
|
|
|
|
|
func logTraffic(ti TrafficInfo) {
|
|
|
|
dataLogChan <- DataLogRow{tbl: "traffic", data: ti}
|
|
|
|
}
|
2016-03-24 04:23:12 +00:00
|
|
|
|
2016-03-24 13:33:11 +00:00
|
|
|
func initDataLog() {
|
|
|
|
go dataLogWriter()
|
2016-03-24 03:08:00 +00:00
|
|
|
}
|