kopia lustrzana https://github.com/ukhas/habitat-cpp-connector
implement numeric scale
rodzic
fffaf112c7
commit
383f1e882b
|
@ -5,6 +5,7 @@
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <cmath>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include "jsoncpp.h"
|
#include "jsoncpp.h"
|
||||||
|
@ -277,6 +278,22 @@ static string convert_ddmmmm(const string &value)
|
||||||
return os.str();
|
return os.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool is_numeric_field(const Json::Value &field)
|
||||||
|
{
|
||||||
|
return field["sensor"] == "base.ascii_int" ||
|
||||||
|
field["sensor"] == "base.ascii_float";
|
||||||
|
}
|
||||||
|
|
||||||
|
static double convert_numeric(const string &value)
|
||||||
|
{
|
||||||
|
istringstream is(value);
|
||||||
|
double val;
|
||||||
|
is >> val;
|
||||||
|
if (is.fail())
|
||||||
|
throw runtime_error("couldn't parse numeric value");
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
static void extract_fields(Json::Value &data, const Json::Value &fields,
|
static void extract_fields(Json::Value &data, const Json::Value &fields,
|
||||||
const vector<string> &parts)
|
const vector<string> &parts)
|
||||||
{
|
{
|
||||||
|
@ -298,6 +315,8 @@ static void extract_fields(Json::Value &data, const Json::Value &fields,
|
||||||
{
|
{
|
||||||
if (is_ddmmmm_field(*field))
|
if (is_ddmmmm_field(*field))
|
||||||
data[key] = convert_ddmmmm(value);
|
data[key] = convert_ddmmmm(value);
|
||||||
|
else if (is_numeric_field(*field))
|
||||||
|
data[key] = convert_numeric(value);
|
||||||
else
|
else
|
||||||
data[key] = value;
|
data[key] = value;
|
||||||
}
|
}
|
||||||
|
@ -307,6 +326,93 @@ static void extract_fields(Json::Value &data, const Json::Value &fields,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void numeric_scale(Json::Value &data, const Json::Value &config)
|
||||||
|
{
|
||||||
|
const string source = config["source"].asString();
|
||||||
|
string destination = source;
|
||||||
|
|
||||||
|
if (!config["destination"].isNull())
|
||||||
|
{
|
||||||
|
if (!config["destination"].isString())
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non string destination)");
|
||||||
|
destination = config["destination"].asString();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (destination == "payload" ||
|
||||||
|
(destination.size() && destination[0] == '_'))
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(forbidden destination)");
|
||||||
|
|
||||||
|
if (!data[source].isNumeric())
|
||||||
|
throw runtime_error("Attempted to apply numeric scale to "
|
||||||
|
"(non numeric source value)");
|
||||||
|
if (!config["factor"].isNumeric())
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non numeric factor)");
|
||||||
|
if (!config["source"].isString())
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non string source)");
|
||||||
|
|
||||||
|
double value = data[source].asDouble();
|
||||||
|
double factor = config["factor"].asDouble();
|
||||||
|
|
||||||
|
value *= factor;
|
||||||
|
|
||||||
|
if (!config["offset"].isNull())
|
||||||
|
{
|
||||||
|
if (!config["offset"].isNumeric())
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non numeric offset)");
|
||||||
|
|
||||||
|
double offset = config["offset"].asDouble();
|
||||||
|
|
||||||
|
value += offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!config["round"].isNull())
|
||||||
|
{
|
||||||
|
if (!config["round"].isNumeric())
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non numeric round)");
|
||||||
|
|
||||||
|
double round_d = config["round"].asDouble();
|
||||||
|
int round_i = int(round_d);
|
||||||
|
|
||||||
|
if (fabs(double(round_i) - round_d) > 0.001)
|
||||||
|
throw runtime_error("Invalid (numeric scale) configuration "
|
||||||
|
"(non integral round)");
|
||||||
|
|
||||||
|
if (value != 0)
|
||||||
|
{
|
||||||
|
int position = round_i - int(ceil(log10(fabs(value))));
|
||||||
|
double m = pow(10, position);
|
||||||
|
value = round(value * m) / m;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data[destination] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void post_filters(Json::Value &data, const Json::Value &sentence)
|
||||||
|
{
|
||||||
|
if (!sentence["filters"].isObject())
|
||||||
|
return;
|
||||||
|
|
||||||
|
const Json::Value post_filters = sentence["filters"]["post"];
|
||||||
|
|
||||||
|
if (!post_filters.isArray())
|
||||||
|
return;
|
||||||
|
|
||||||
|
for (Json::Value::const_iterator it = post_filters.begin();
|
||||||
|
it != post_filters.end(); it++)
|
||||||
|
{
|
||||||
|
if ((*it)["type"] == "normal" &&
|
||||||
|
(*it)["filter"] == "common.numeric_scale")
|
||||||
|
numeric_scale(data, *it);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static void cook_basic(Json::Value &basic, const string &buffer,
|
static void cook_basic(Json::Value &basic, const string &buffer,
|
||||||
const string &callsign)
|
const string &callsign)
|
||||||
{
|
{
|
||||||
|
@ -337,6 +443,7 @@ static void attempt_settings(Json::Value &data, const Json::Value &sentence,
|
||||||
throw runtime_error("Incorrect number of fields");
|
throw runtime_error("Incorrect number of fields");
|
||||||
|
|
||||||
extract_fields(data, fields, parts);
|
extract_fields(data, fields, parts);
|
||||||
|
post_filters(data, sentence);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* crude_parse is based on the parse() method of
|
/* crude_parse is based on the parse() method of
|
||||||
|
@ -376,7 +483,7 @@ Json::Value UKHASExtractor::crude_parse()
|
||||||
/* Silence errors, and only log them if all attempts fail */
|
/* Silence errors, and only log them if all attempts fail */
|
||||||
vector<string> errors;
|
vector<string> errors;
|
||||||
|
|
||||||
for (Json::Value::iterator it = sentences.begin();
|
for (Json::Value::const_iterator it = sentences.begin();
|
||||||
it != sentences.end(); it++)
|
it != sentences.end(); it++)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
@ -393,7 +500,7 @@ Json::Value UKHASExtractor::crude_parse()
|
||||||
|
|
||||||
/* Couldn't parse using any of the settings... */
|
/* Couldn't parse using any of the settings... */
|
||||||
mgr->status("UKHAS Extractor: full parse failed:");
|
mgr->status("UKHAS Extractor: full parse failed:");
|
||||||
for (vector<string>::iterator it = errors.begin();
|
for (vector<string>::const_iterator it = errors.begin();
|
||||||
it != errors.end(); it++)
|
it != errors.end(); it++)
|
||||||
{
|
{
|
||||||
mgr->status("UKHAS Extractor: " + (*it));
|
mgr->status("UKHAS Extractor: " + (*it));
|
||||||
|
|
|
@ -15,6 +15,47 @@ class EqualIfIn:
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return "<EqIn " + repr(self.test) + ">"
|
return "<EqIn " + repr(self.test) + ">"
|
||||||
|
|
||||||
|
# from habitat.views.payload_telemetry:
|
||||||
|
def is_equal_relaxed_floats(a, b):
|
||||||
|
"""
|
||||||
|
Check that a == b, allowing small float differences
|
||||||
|
"""
|
||||||
|
|
||||||
|
if isinstance(a, list) or isinstance(a, dict):
|
||||||
|
# recursion
|
||||||
|
if isinstance(a, list):
|
||||||
|
if not isinstance(b, list):
|
||||||
|
return False
|
||||||
|
keys_iter = xrange(len(a))
|
||||||
|
else:
|
||||||
|
if not isinstance(b, dict):
|
||||||
|
return False
|
||||||
|
keys_iter = a
|
||||||
|
|
||||||
|
if len(a) != len(b):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return all(is_equal_relaxed_floats(a[i], b[i]) for i in keys_iter)
|
||||||
|
|
||||||
|
elif isinstance(a, float) or isinstance(b, float):
|
||||||
|
if not (isinstance(a, float) or isinstance(a, int)) or \
|
||||||
|
not (isinstance(b, float) or isinstance(b, int)):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# fast path
|
||||||
|
if a == b:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# relaxed float comparison.
|
||||||
|
# Doubles provide 15-17 bits of precision. Converting to decimal and
|
||||||
|
# back should not introduce an error larger than 1e-15, really.
|
||||||
|
tolerance = max(a, b) * 1e-14
|
||||||
|
return abs(a - b) < tolerance
|
||||||
|
|
||||||
|
else:
|
||||||
|
# string, int, bool, None, ...
|
||||||
|
return a == b
|
||||||
|
|
||||||
class Proxy:
|
class Proxy:
|
||||||
def __init__(self, command):
|
def __init__(self, command):
|
||||||
self.closed = False
|
self.closed = False
|
||||||
|
@ -69,7 +110,7 @@ class Proxy:
|
||||||
def check(self, match):
|
def check(self, match):
|
||||||
obj = self._read()
|
obj = self._read()
|
||||||
assert len(obj) >= len(match)
|
assert len(obj) >= len(match)
|
||||||
assert obj[:len(match)] == match
|
assert is_equal_relaxed_floats(obj[:len(match)], match)
|
||||||
|
|
||||||
def _check_type(self, name, arg):
|
def _check_type(self, name, arg):
|
||||||
if arg:
|
if arg:
|
||||||
|
@ -238,14 +279,16 @@ class TestUKHASExtractor:
|
||||||
"fields": [
|
"fields": [
|
||||||
{"name": "field_a"},
|
{"name": "field_a"},
|
||||||
{"name": "field_b"},
|
{"name": "field_b"},
|
||||||
{"name": "field_c"}
|
{"name": "field_c"},
|
||||||
|
{"name": "int_d", "sensor": "base.ascii_int"},
|
||||||
|
{"name": "float_e", "sensor": "base.ascii_float"},
|
||||||
],
|
],
|
||||||
} ]
|
} ]
|
||||||
}
|
}
|
||||||
|
|
||||||
def test_crude_parse_config(self):
|
def test_crude_parse_config(self):
|
||||||
self.extr.set_current_payload(self.crude_parse_flight_doc)
|
self.extr.set_current_payload(self.crude_parse_flight_doc)
|
||||||
string = "$$TESTING,value_a,value_b,value_c*8C3E\n"
|
string = "$$TESTING,value_a,value_b,value_c,123,453.24*CC76\n"
|
||||||
self.extr.push(string)
|
self.extr.push(string)
|
||||||
self.extr.check_status("start delim")
|
self.extr.check_status("start delim")
|
||||||
self.extr.check_upload(string)
|
self.extr.check_upload(string)
|
||||||
|
@ -253,7 +296,8 @@ class TestUKHASExtractor:
|
||||||
self.extr.check_data({"_sentence": string, "_parsed": True,
|
self.extr.check_data({"_sentence": string, "_parsed": True,
|
||||||
"_protocol": "UKHAS", "payload": "TESTING",
|
"_protocol": "UKHAS", "payload": "TESTING",
|
||||||
"field_a": "value_a", "field_b": "value_b",
|
"field_a": "value_a", "field_b": "value_b",
|
||||||
"field_c": "value_c"})
|
"field_c": "value_c", "int_d": 123,
|
||||||
|
"float_e": 453.24})
|
||||||
|
|
||||||
def test_crude_checks(self):
|
def test_crude_checks(self):
|
||||||
checks = [
|
checks = [
|
||||||
|
@ -335,3 +379,41 @@ class TestUKHASExtractor:
|
||||||
"_protocol": "UKHAS", "payload": "TESTING",
|
"_protocol": "UKHAS", "payload": "TESTING",
|
||||||
"lat_a": "0024.124583", "lat_b": "51.27545",
|
"lat_a": "0024.124583", "lat_b": "51.27545",
|
||||||
"field_b": "whatever" })
|
"field_b": "whatever" })
|
||||||
|
|
||||||
|
numeric_scale_flight_doc = {
|
||||||
|
"sentences": [ {
|
||||||
|
"callsign": "TESTING",
|
||||||
|
"checksum": "crc16-ccitt",
|
||||||
|
"fields": [
|
||||||
|
{"sensor":"base.ascii_float","name":"a"},
|
||||||
|
{"sensor":"base.ascii_float","name":"b"},
|
||||||
|
{"sensor":"base.ascii_float","name":"c"}
|
||||||
|
],
|
||||||
|
"filters": {
|
||||||
|
"post": [
|
||||||
|
{"filter": "un.related", "type": "normal",
|
||||||
|
"some config": True},
|
||||||
|
{"filter": "common.numeric_scale", "type": "normal",
|
||||||
|
"source": "a", "offset": 6, "factor": 2, "round": 3},
|
||||||
|
{"type": "hotfix", "ignore me": True},
|
||||||
|
{"filter": "common.numeric_scale", "type": "normal",
|
||||||
|
"source": "b", "destination": "b2", "factor": 0.001,
|
||||||
|
"round": 3},
|
||||||
|
{"filter": "common.numeric_scale", "type": "normal",
|
||||||
|
"source": "b", "destination": "b3", "factor": 5}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
} ]
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_numeric_scale(self):
|
||||||
|
self.extr.set_current_payload(self.numeric_scale_flight_doc)
|
||||||
|
string = "$$TESTING,100.123,0.00482123,48*60A4\n"
|
||||||
|
self.extr.push(string)
|
||||||
|
self.extr.check_status("start delim")
|
||||||
|
self.extr.check_upload(string)
|
||||||
|
self.extr.check_status("extracted")
|
||||||
|
self.extr.check_data({"_sentence": string, "_parsed": True,
|
||||||
|
"_protocol": "UKHAS", "payload": "TESTING",
|
||||||
|
"a": 206, "b": 0.00482123, "b2": 0.00000482,
|
||||||
|
"b3": 0.00482123 * 5, "c": 48})
|
||||||
|
|
Ładowanie…
Reference in New Issue