diff --git a/src/UKHASExtractor.cxx b/src/UKHASExtractor.cxx index 0155c1c..a196207 100644 --- a/src/UKHASExtractor.cxx +++ b/src/UKHASExtractor.cxx @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include "jsoncpp.h" @@ -277,6 +278,22 @@ static string convert_ddmmmm(const string &value) return os.str(); } +static bool is_numeric_field(const Json::Value &field) +{ + return field["sensor"] == "base.ascii_int" || + field["sensor"] == "base.ascii_float"; +} + +static double convert_numeric(const string &value) +{ + istringstream is(value); + double val; + is >> val; + if (is.fail()) + throw runtime_error("couldn't parse numeric value"); + return val; +} + static void extract_fields(Json::Value &data, const Json::Value &fields, const vector &parts) { @@ -298,6 +315,8 @@ static void extract_fields(Json::Value &data, const Json::Value &fields, { if (is_ddmmmm_field(*field)) data[key] = convert_ddmmmm(value); + else if (is_numeric_field(*field)) + data[key] = convert_numeric(value); else data[key] = value; } @@ -307,6 +326,93 @@ static void extract_fields(Json::Value &data, const Json::Value &fields, } } +static void numeric_scale(Json::Value &data, const Json::Value &config) +{ + const string source = config["source"].asString(); + string destination = source; + + if (!config["destination"].isNull()) + { + if (!config["destination"].isString()) + throw runtime_error("Invalid (numeric scale) configuration " + "(non string destination)"); + destination = config["destination"].asString(); + } + + if (destination == "payload" || + (destination.size() && destination[0] == '_')) + throw runtime_error("Invalid (numeric scale) configuration " + "(forbidden destination)"); + + if (!data[source].isNumeric()) + throw runtime_error("Attempted to apply numeric scale to " + "(non numeric source value)"); + if (!config["factor"].isNumeric()) + throw runtime_error("Invalid (numeric scale) configuration " + "(non numeric factor)"); + if (!config["source"].isString()) + throw runtime_error("Invalid (numeric scale) configuration " + "(non string source)"); + + double value = data[source].asDouble(); + double factor = config["factor"].asDouble(); + + value *= factor; + + if (!config["offset"].isNull()) + { + if (!config["offset"].isNumeric()) + throw runtime_error("Invalid (numeric scale) configuration " + "(non numeric offset)"); + + double offset = config["offset"].asDouble(); + + value += offset; + } + + if (!config["round"].isNull()) + { + if (!config["round"].isNumeric()) + throw runtime_error("Invalid (numeric scale) configuration " + "(non numeric round)"); + + double round_d = config["round"].asDouble(); + int round_i = int(round_d); + + if (fabs(double(round_i) - round_d) > 0.001) + throw runtime_error("Invalid (numeric scale) configuration " + "(non integral round)"); + + if (value != 0) + { + int position = round_i - int(ceil(log10(fabs(value)))); + double m = pow(10, position); + value = round(value * m) / m; + } + } + + data[destination] = value; +} + +static void post_filters(Json::Value &data, const Json::Value &sentence) +{ + if (!sentence["filters"].isObject()) + return; + + const Json::Value post_filters = sentence["filters"]["post"]; + + if (!post_filters.isArray()) + return; + + for (Json::Value::const_iterator it = post_filters.begin(); + it != post_filters.end(); it++) + { + if ((*it)["type"] == "normal" && + (*it)["filter"] == "common.numeric_scale") + numeric_scale(data, *it); + } +} + static void cook_basic(Json::Value &basic, const string &buffer, const string &callsign) { @@ -337,6 +443,7 @@ static void attempt_settings(Json::Value &data, const Json::Value &sentence, throw runtime_error("Incorrect number of fields"); extract_fields(data, fields, parts); + post_filters(data, sentence); } /* crude_parse is based on the parse() method of @@ -376,7 +483,7 @@ Json::Value UKHASExtractor::crude_parse() /* Silence errors, and only log them if all attempts fail */ vector errors; - for (Json::Value::iterator it = sentences.begin(); + for (Json::Value::const_iterator it = sentences.begin(); it != sentences.end(); it++) { try @@ -393,7 +500,7 @@ Json::Value UKHASExtractor::crude_parse() /* Couldn't parse using any of the settings... */ mgr->status("UKHAS Extractor: full parse failed:"); - for (vector::iterator it = errors.begin(); + for (vector::const_iterator it = errors.begin(); it != errors.end(); it++) { mgr->status("UKHAS Extractor: " + (*it)); diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 74f3063..4504e34 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -15,6 +15,47 @@ class EqualIfIn: def __repr__(self): return "" +# from habitat.views.payload_telemetry: +def is_equal_relaxed_floats(a, b): + """ + Check that a == b, allowing small float differences + """ + + if isinstance(a, list) or isinstance(a, dict): + # recursion + if isinstance(a, list): + if not isinstance(b, list): + return False + keys_iter = xrange(len(a)) + else: + if not isinstance(b, dict): + return False + keys_iter = a + + if len(a) != len(b): + return False + + return all(is_equal_relaxed_floats(a[i], b[i]) for i in keys_iter) + + elif isinstance(a, float) or isinstance(b, float): + if not (isinstance(a, float) or isinstance(a, int)) or \ + not (isinstance(b, float) or isinstance(b, int)): + return False + + # fast path + if a == b: + return True + + # relaxed float comparison. + # Doubles provide 15-17 bits of precision. Converting to decimal and + # back should not introduce an error larger than 1e-15, really. + tolerance = max(a, b) * 1e-14 + return abs(a - b) < tolerance + + else: + # string, int, bool, None, ... + return a == b + class Proxy: def __init__(self, command): self.closed = False @@ -69,7 +110,7 @@ class Proxy: def check(self, match): obj = self._read() assert len(obj) >= len(match) - assert obj[:len(match)] == match + assert is_equal_relaxed_floats(obj[:len(match)], match) def _check_type(self, name, arg): if arg: @@ -238,14 +279,16 @@ class TestUKHASExtractor: "fields": [ {"name": "field_a"}, {"name": "field_b"}, - {"name": "field_c"} + {"name": "field_c"}, + {"name": "int_d", "sensor": "base.ascii_int"}, + {"name": "float_e", "sensor": "base.ascii_float"}, ], } ] } def test_crude_parse_config(self): self.extr.set_current_payload(self.crude_parse_flight_doc) - string = "$$TESTING,value_a,value_b,value_c*8C3E\n" + string = "$$TESTING,value_a,value_b,value_c,123,453.24*CC76\n" self.extr.push(string) self.extr.check_status("start delim") self.extr.check_upload(string) @@ -253,7 +296,8 @@ class TestUKHASExtractor: self.extr.check_data({"_sentence": string, "_parsed": True, "_protocol": "UKHAS", "payload": "TESTING", "field_a": "value_a", "field_b": "value_b", - "field_c": "value_c"}) + "field_c": "value_c", "int_d": 123, + "float_e": 453.24}) def test_crude_checks(self): checks = [ @@ -335,3 +379,41 @@ class TestUKHASExtractor: "_protocol": "UKHAS", "payload": "TESTING", "lat_a": "0024.124583", "lat_b": "51.27545", "field_b": "whatever" }) + + numeric_scale_flight_doc = { + "sentences": [ { + "callsign": "TESTING", + "checksum": "crc16-ccitt", + "fields": [ + {"sensor":"base.ascii_float","name":"a"}, + {"sensor":"base.ascii_float","name":"b"}, + {"sensor":"base.ascii_float","name":"c"} + ], + "filters": { + "post": [ + {"filter": "un.related", "type": "normal", + "some config": True}, + {"filter": "common.numeric_scale", "type": "normal", + "source": "a", "offset": 6, "factor": 2, "round": 3}, + {"type": "hotfix", "ignore me": True}, + {"filter": "common.numeric_scale", "type": "normal", + "source": "b", "destination": "b2", "factor": 0.001, + "round": 3}, + {"filter": "common.numeric_scale", "type": "normal", + "source": "b", "destination": "b3", "factor": 5} + ] + } + } ] + } + + def test_numeric_scale(self): + self.extr.set_current_payload(self.numeric_scale_flight_doc) + string = "$$TESTING,100.123,0.00482123,48*60A4\n" + self.extr.push(string) + self.extr.check_status("start delim") + self.extr.check_upload(string) + self.extr.check_status("extracted") + self.extr.check_data({"_sentence": string, "_parsed": True, + "_protocol": "UKHAS", "payload": "TESTING", + "a": 206, "b": 0.00482123, "b2": 0.00000482, + "b3": 0.00482123 * 5, "c": 48})