mirror
/
habitat-cpp-connector
kopia lustrzana https://github.com/ukhas/habitat-cpp-connector
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

implement numeric scale

pull/2/head
Daniel Richman 2013-08-11 12:21:05 +01:00
rodzic fffaf112c7
commit 383f1e882b
2 zmienionych plików z 195 dodań i 6 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

111
src/UKHASExtractor.cxx
Wyświetl plik

@ -5,6 +5,7 @@
#include <string>
#include <sstream>
#include <algorithm>
#include <cmath>
#include <stdio.h>
#include <stdint.h>
#include "jsoncpp.h"
@ -277,6 +278,22 @@ static string convert_ddmmmm(const string &value)
return os.str();
}
static bool is_numeric_field(const Json::Value &field)
{
return field["sensor"] == "base.ascii_int" ||
field["sensor"] == "base.ascii_float";
}
static double convert_numeric(const string &value)
{
istringstream is(value);
double val;
is >> val;
if (is.fail())
throw runtime_error("couldn't parse numeric value");
return val;
}
static void extract_fields(Json::Value &data, const Json::Value &fields,
const vector<string> &parts)
{
@ -298,6 +315,8 @@ static void extract_fields(Json::Value &data, const Json::Value &fields,
{
if (is_ddmmmm_field(*field))
data[key] = convert_ddmmmm(value);
else if (is_numeric_field(*field))
data[key] = convert_numeric(value);
else
data[key] = value;
}
@ -307,6 +326,93 @@ static void extract_fields(Json::Value &data, const Json::Value &fields,
}
}
static void numeric_scale(Json::Value &data, const Json::Value &config)
{
const string source = config["source"].asString();
string destination = source;
if (!config["destination"].isNull())
{
if (!config["destination"].isString())
throw runtime_error("Invalid (numeric scale) configuration "
"(non string destination)");
destination = config["destination"].asString();
}
if (destination == "payload" ||
(destination.size() && destination[0] == '_'))
throw runtime_error("Invalid (numeric scale) configuration "
"(forbidden destination)");
if (!data[source].isNumeric())
throw runtime_error("Attempted to apply numeric scale to "
"(non numeric source value)");
if (!config["factor"].isNumeric())
throw runtime_error("Invalid (numeric scale) configuration "
"(non numeric factor)");
if (!config["source"].isString())
throw runtime_error("Invalid (numeric scale) configuration "
"(non string source)");
double value = data[source].asDouble();
double factor = config["factor"].asDouble();
value *= factor;
if (!config["offset"].isNull())
{
if (!config["offset"].isNumeric())
throw runtime_error("Invalid (numeric scale) configuration "
"(non numeric offset)");
double offset = config["offset"].asDouble();
value += offset;
}
if (!config["round"].isNull())
{
if (!config["round"].isNumeric())
throw runtime_error("Invalid (numeric scale) configuration "
"(non numeric round)");
double round_d = config["round"].asDouble();
int round_i = int(round_d);
if (fabs(double(round_i) - round_d) > 0.001)
throw runtime_error("Invalid (numeric scale) configuration "
"(non integral round)");
if (value != 0)
{
int position = round_i - int(ceil(log10(fabs(value))));
double m = pow(10, position);
value = round(value * m) / m;
}
}
data[destination] = value;
}
static void post_filters(Json::Value &data, const Json::Value &sentence)
{
if (!sentence["filters"].isObject())
return;
const Json::Value post_filters = sentence["filters"]["post"];
if (!post_filters.isArray())
return;
for (Json::Value::const_iterator it = post_filters.begin();
it != post_filters.end(); it++)
{
if ((*it)["type"] == "normal" &&
(*it)["filter"] == "common.numeric_scale")
numeric_scale(data, *it);
}
}
static void cook_basic(Json::Value &basic, const string &buffer,
const string &callsign)
{
@ -337,6 +443,7 @@ static void attempt_settings(Json::Value &data, const Json::Value &sentence,
throw runtime_error("Incorrect number of fields");
extract_fields(data, fields, parts);
post_filters(data, sentence);
}
/* crude_parse is based on the parse() method of
@ -376,7 +483,7 @@ Json::Value UKHASExtractor::crude_parse()
/* Silence errors, and only log them if all attempts fail */
vector<string> errors;
for (Json::Value::iterator it = sentences.begin();
for (Json::Value::const_iterator it = sentences.begin();
it != sentences.end(); it++)
{
try
@ -393,7 +500,7 @@ Json::Value UKHASExtractor::crude_parse()
/* Couldn't parse using any of the settings... */
mgr->status("UKHAS Extractor: full parse failed:");
for (vector<string>::iterator it = errors.begin();
for (vector<string>::const_iterator it = errors.begin();
it != errors.end(); it++)
{
mgr->status("UKHAS Extractor: " + (*it));

90
tests/test_extractor.py
Wyświetl plik

@ -15,6 +15,47 @@ class EqualIfIn:
def __repr__(self):
return "<EqIn " + repr(self.test) + ">"
# from habitat.views.payload_telemetry:
def is_equal_relaxed_floats(a, b):
"""
Check that a == b, allowing small float differences
"""
if isinstance(a, list) or isinstance(a, dict):
# recursion
if isinstance(a, list):
if not isinstance(b, list):
return False
keys_iter = xrange(len(a))
else:
if not isinstance(b, dict):
return False
keys_iter = a
if len(a) != len(b):
return False
return all(is_equal_relaxed_floats(a[i], b[i]) for i in keys_iter)
elif isinstance(a, float) or isinstance(b, float):
if not (isinstance(a, float) or isinstance(a, int)) or \
not (isinstance(b, float) or isinstance(b, int)):
return False
# fast path
if a == b:
return True
# relaxed float comparison.
# Doubles provide 15-17 bits of precision. Converting to decimal and
# back should not introduce an error larger than 1e-15, really.
tolerance = max(a, b) * 1e-14
return abs(a - b) < tolerance
else:
# string, int, bool, None, ...
return a == b
class Proxy:
def __init__(self, command):
self.closed = False
@ -69,7 +110,7 @@ class Proxy:
def check(self, match):
obj = self._read()
assert len(obj) >= len(match)
assert obj[:len(match)] == match
assert is_equal_relaxed_floats(obj[:len(match)], match)
def _check_type(self, name, arg):
if arg:
@ -238,14 +279,16 @@ class TestUKHASExtractor:
"fields": [
{"name": "field_a"},
{"name": "field_b"},
{"name": "field_c"}
{"name": "field_c"},
{"name": "int_d", "sensor": "base.ascii_int"},
{"name": "float_e", "sensor": "base.ascii_float"},
],
} ]
}
def test_crude_parse_config(self):
self.extr.set_current_payload(self.crude_parse_flight_doc)
string = "$$TESTING,value_a,value_b,value_c*8C3E\n"
string = "$$TESTING,value_a,value_b,value_c,123,453.24*CC76\n"
self.extr.push(string)
self.extr.check_status("start delim")
self.extr.check_upload(string)
@ -253,7 +296,8 @@ class TestUKHASExtractor:
self.extr.check_data({"_sentence": string, "_parsed": True,
"_protocol": "UKHAS", "payload": "TESTING",
"field_a": "value_a", "field_b": "value_b",
"field_c": "value_c"})
"field_c": "value_c", "int_d": 123,
"float_e": 453.24})
def test_crude_checks(self):
checks = [
@ -335,3 +379,41 @@ class TestUKHASExtractor:
"_protocol": "UKHAS", "payload": "TESTING",
"lat_a": "0024.124583", "lat_b": "51.27545",
"field_b": "whatever" })
numeric_scale_flight_doc = {
"sentences": [ {
"callsign": "TESTING",
"checksum": "crc16-ccitt",
"fields": [
{"sensor":"base.ascii_float","name":"a"},
{"sensor":"base.ascii_float","name":"b"},
{"sensor":"base.ascii_float","name":"c"}
],
"filters": {
"post": [
{"filter": "un.related", "type": "normal",
"some config": True},
{"filter": "common.numeric_scale", "type": "normal",
"source": "a", "offset": 6, "factor": 2, "round": 3},
{"type": "hotfix", "ignore me": True},
{"filter": "common.numeric_scale", "type": "normal",
"source": "b", "destination": "b2", "factor": 0.001,
"round": 3},
{"filter": "common.numeric_scale", "type": "normal",
"source": "b", "destination": "b3", "factor": 5}
]
}
} ]
}
def test_numeric_scale(self):
self.extr.set_current_payload(self.numeric_scale_flight_doc)
string = "$$TESTING,100.123,0.00482123,48*60A4\n"
self.extr.push(string)
self.extr.check_status("start delim")
self.extr.check_upload(string)
self.extr.check_status("extracted")
self.extr.check_data({"_sentence": string, "_parsed": True,
"_protocol": "UKHAS", "payload": "TESTING",
"a": 206, "b": 0.00482123, "b2": 0.00000482,
"b3": 0.00482123 * 5, "c": 48})