From 6823279b05a68d378b8f07004a87ce7320ebef9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20=C3=96hrstr=C3=B6m?= Date: Sun, 6 Feb 2022 18:49:55 +0100 Subject: [PATCH] Significantly improve --analyze. --- src/cmdline.cc | 49 +++- src/config.h | 3 + src/driver_apator08.cc | 11 +- src/main.cc | 6 +- src/meter_apator162.cc | 4 +- src/meters.cc | 415 +++++++++++++++++++---------- src/meters.h | 2 +- src/meters_common_implementation.h | 8 + src/wmbus.cc | 100 +++++-- src/wmbus.h | 6 +- src/wmbus_utils.cc | 34 ++- src/wmbus_utils.h | 9 +- test.sh | 4 +- tests/test_analyze.sh | 350 ++++++++++++++++-------- 14 files changed, 692 insertions(+), 309 deletions(-) diff --git a/src/cmdline.cc b/src/cmdline.cc index 4270f90..ab952b4 100644 --- a/src/cmdline.cc +++ b/src/cmdline.cc @@ -95,15 +95,56 @@ static shared_ptr parseNormalCommandLine(Configuration *c, int ar { c->analyze_format = OutputFormat::PLAIN; } + c->analyze_driver = ""; + c->analyze_key = ""; + c->analyze_verbose = false; i++; continue; } if (!strncmp(argv[i], "--analyze=", 10)) { c->analyze = true; - string format = string(argv[i]+10); - if (format == "plain") c->analyze_format = OutputFormat::PLAIN; - else if (format == "terminal") c->analyze_format = OutputFormat::TERMINAL; - else if (format == "json") c->analyze_format = OutputFormat::JSON; + if (isatty(1)) + { + c->analyze_format = OutputFormat::TERMINAL; + } + else + { + c->analyze_format = OutputFormat::PLAIN; + } + c->analyze_driver = ""; + c->analyze_key = ""; + c->analyze_verbose = false; + string arg = string(argv[i]+10); + vector args = splitString(arg, ':'); + + for (auto s : args) + { + bool inv = false; + if (isHexStringStrict(s, &inv)) + { + if (inv) + { + error("Bad key \"%s\"", s.c_str()); + } + c->analyze_key = s; + } + else if (s == "plain") c->analyze_format = OutputFormat::PLAIN; + else if (s == "terminal") c->analyze_format = OutputFormat::TERMINAL; + else if (s == "json") c->analyze_format = OutputFormat::JSON; + else if (s == "verbose") c->analyze_verbose = true; + else + { + MeterInfo mi; + mi.parse("x", s, "00000000", ""); + + if (mi.driver == MeterDriver::UNKNOWN && + mi.driver_name.str() == "") + { + error("Not a valid meter driver \"%s\"\n", s.c_str()); + } + c->analyze_driver = s; + } + } i++; continue; } diff --git a/src/config.h b/src/config.h index df06de1..2b0b1d5 100644 --- a/src/config.h +++ b/src/config.h @@ -70,6 +70,9 @@ struct Configuration bool license {}; bool analyze {}; OutputFormat analyze_format {}; + string analyze_driver {}; + string analyze_key {}; + bool analyze_verbose {}; bool debug {}; bool trace {}; AddLogTimestamps addtimestamps {}; diff --git a/src/driver_apator08.cc b/src/driver_apator08.cc index df19dec..e273fd6 100644 --- a/src/driver_apator08.cc +++ b/src/driver_apator08.cc @@ -42,16 +42,9 @@ static bool ok = registerDriver([](DriverInfo&di) MeterApator08::MeterApator08(MeterInfo &mi, DriverInfo &di) : MeterCommonImplementation(mi, di) { - addFieldWithExtractor( + addField( "total", Quantity::Volume, - NoDifVifKey, - VifScaling::Auto, - MeasurementType::Instantaneous, - ValueInformation::Volume, - StorageNr(0), - TariffNr(0), - IndexNr(1), PrintProperty::JSON | PrintProperty::FIELD | PrintProperty::IMPORTANT, "The total water consumption recorded by this meter.", SET_FUNC(total_water_consumption_m3_, Unit::M3), @@ -84,7 +77,7 @@ void MeterApator08::processContent(Telegram *t) total_water_consumption_m3_ /= 3.0; total = "*** 10|"+total+" total consumption (%f m3)"; - t->addSpecialExplanation(offset, KindOfData::CONTENT, Understanding::FULL, total.c_str(), total_water_consumption_m3_); + t->addSpecialExplanation(offset, 4, KindOfData::CONTENT, Understanding::FULL, total.c_str(), total_water_consumption_m3_); } } diff --git a/src/main.cc b/src/main.cc index b2ca807..a4656b5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -497,7 +497,11 @@ bool start(Configuration *config) // and creates meters on demand when the telegram arrives // or on startup for 2-way communication meters like mbus or T2. meter_manager_ = createMeterManager(config->daemon); - meter_manager_->analyzeEnabled(config->analyze, config->analyze_format); + meter_manager_->analyzeEnabled(config->analyze, + config->analyze_format, + config->analyze_driver, + config->analyze_key, + config->analyze_verbose); // The bus manager detects new/lost wmbus devices and // configures the devices according to the specification. diff --git a/src/meter_apator162.cc b/src/meter_apator162.cc index 6b7e646..dc7c9ad 100644 --- a/src/meter_apator162.cc +++ b/src/meter_apator162.cc @@ -129,13 +129,13 @@ void MeterApator162::processContent(Telegram *t) int offset; extractDVdouble(&vendor_values, "0413", &offset, &total_water_consumption_m3_); total = "*** 10|"+total+" total consumption (%f m3)"; - t->addSpecialExplanation(offset, KindOfData::CONTENT, Understanding::FULL, total.c_str(), total_water_consumption_m3_); + t->addSpecialExplanation(offset, 4, KindOfData::CONTENT, Understanding::FULL, total.c_str(), total_water_consumption_m3_); } else { string msg = "*** "; msg += bin2hex(content, i-1, 1)+"|"+bin2hex(content, i, size); - t->addSpecialExplanation(i-1+t->header_size, KindOfData::CONTENT, Understanding::NONE, msg.c_str()); + t->addSpecialExplanation(i-1+t->header_size, size, KindOfData::CONTENT, Understanding::NONE, msg.c_str()); } i += size; } diff --git a/src/meters.cc b/src/meters.cc index e8bf7f8..1448c4a 100644 --- a/src/meters.cc +++ b/src/meters.cc @@ -108,6 +108,9 @@ private: bool is_daemon_ {}; bool should_analyze_ {}; OutputFormat analyze_format_ {}; + string analyze_driver_; + string analyze_key_; + bool analyze_verbose_; vector meter_templates_; vector> meters_; function)> on_telegram_; @@ -343,16 +346,145 @@ public: } } - void analyzeEnabled(bool b, OutputFormat f) + void analyzeEnabled(bool b, OutputFormat f, string force_driver, string key, bool verbose) { should_analyze_ = b; analyze_format_ = f; + analyze_driver_ = force_driver; + analyze_key_ = key; + analyze_verbose_ = verbose; + } + + string findBestOldStyleDriver(MeterInfo &mi, + int *best_length, + int *best_understood, + Telegram &t, + AboutTelegram &about, + vector &input_frame, + bool simulated, + string only) + { + vector old_drivers; +#define X(mname,linkmode,info,type,cname) old_drivers.push_back(MeterDriver::type); +LIST_OF_METERS +#undef X + + string best_driver = ""; + for (MeterDriver odr : old_drivers) + { + if (odr == MeterDriver::AUTO) continue; + if (odr == MeterDriver::UNKNOWN) continue; + string driver_name = toString(odr); + if (only != "" && driver_name != only) continue; + + if (!isMeterDriverReasonableForMedia(odr, "", t.dll_type) && + !isMeterDriverReasonableForMedia(odr, "", t.tpl_type)) + { + // Sanity check, skip this driver since it is not relevant for this media. + continue; + } + + + debug("Testing old style driver %s...\n", driver_name.c_str()); + mi.driver = odr; + mi.driver_name = DriverName(""); + + auto meter = createMeter(&mi); + + bool match = false; + string id; + bool h = meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); + if (!match) + { + debug("no match!\n"); + } + else if (!h) + { + // Oups, we added a new meter object tailored for this telegram + // but it still did not handle it! This can happen if the wrong + // decryption key was used. But it is ok if analyzing.... + debug("Newly created meter (%s %s %s) did not handle telegram!\n", + meter->name().c_str(), meter->idsc().c_str(), meter->driverName().str().c_str()); + } + else + { + int l = 0; + int u = 0; + t.analyzeParse(OutputFormat::NONE, &l, &u); + if (analyze_verbose_ && only == "") printf("(verbose) old %02d/%02d %s\n", u, l, driver_name.c_str()); + if (u > *best_understood) + { + *best_understood = u; + *best_length = l; + best_driver = driver_name; + if (analyze_verbose_ && only == "") printf("(verbose) old best so far: %s %02d/%02d\n", best_driver.c_str(), u, l); + } + } + } + return best_driver; + } + + string findBestNewStyleDriver(MeterInfo &mi, + int *best_length, + int *best_understood, + Telegram &t, + AboutTelegram &about, + vector &input_frame, + bool simulated, + string only) + { + string best_driver = ""; + + for (DriverInfo ndr : all_registered_drivers_list_) + { + string driver_name = toString(ndr); + if (only != "" && driver_name != only) continue; + + debug("Testing new style driver %s...\n", driver_name.c_str()); + mi.driver = MeterDriver::UNKNOWN; + mi.driver_name = driver_name; + + auto meter = createMeter(&mi); + + bool match = false; + string id; + bool h = meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); + + if (!match) + { + debug("no match!\n"); + } + else if (!h) + { + // Oups, we added a new meter object tailored for this telegram + // but it still did not handle it! This can happen if the wrong + // decryption key was used. But it is ok if analyzing.... + debug("Newly created meter (%s %s %s) did not handle telegram!\n", + meter->name().c_str(), meter->idsc().c_str(), meter->driverName().str().c_str()); + } + else + { + int l = 0; + int u = 0; + t.analyzeParse(OutputFormat::NONE, &l, &u); + if (analyze_verbose_ && only == "") printf("(verbose) new %02d/%02d %s\n", u, l, driver_name.c_str()); + if (u > *best_understood) + { + *best_understood = u; + *best_length = l; + best_driver = ndr.name().str(); + if (analyze_verbose_ && only == "") printf("(verbose) new best so far: %s %02d/%02d\n", best_driver.c_str(), u, l); + } + } + } + return best_driver; } void analyzeTelegram(AboutTelegram &about, vector &input_frame, bool simulated) { Telegram t; t.about = about; + bool ok = t.parseHeader(input_frame); if (simulated) t.markAsSimulated(); t.markAsBeingAnalyzed(); @@ -363,168 +495,142 @@ public: return; } - vector drivers; -#define X(mname,linkmode,info,type,cname) drivers.push_back(MeterDriver::type); -LIST_OF_METERS -#undef X - - MeterInfo mi; if (meter_templates_.size() > 0) { - if (meter_templates_.size() > 1) - { - error("When analyzing you can only specify a single meter quadruple.\n"); - } - if (meter_templates_[0].driver != MeterDriver::AUTO) - { - drivers.clear(); - drivers.push_back(meter_templates_[0].driver); - mi = meter_templates_[0]; - } + error("You cannot specify a meter quadruple when analyzing.\n" + "Instead use --analyze=::\n" + "where are all optional.\n" + "E.g. --analyze=terminal:multical21:001122334455667788001122334455667788\n" + " --analyze=001122334455667788001122334455667788\n" + " --analyze\n"); } // Overwrite the id with the id from the telegram to be analyzed. + MeterInfo mi; + mi.key = analyze_key_; mi.ids.clear(); mi.ids.push_back(t.ids.back()); mi.idsc = t.ids.back(); - bool handled = false; - DriverInfo best_driver; - // For the best driver we have: - int best_content_length = 0; - int best_understood_content_length = 0; + // This will be the driver that will actually decode and print with. + string using_driver = ""; + int using_length = 0; + int using_understood = 0; - for (MeterDriver dr : drivers) + // Driver that understands most of the telegram content. + string best_driver = ""; + int best_length = 0; + int best_understood = 0; + + int old_best_length = 0; + int old_best_understood = 0; + string best_old_driver = findBestOldStyleDriver(mi, &old_best_length, &old_best_understood, t, about, input_frame, simulated, ""); + + int new_best_length = 0; + int new_best_understood = 0; + string best_new_driver = findBestNewStyleDriver(mi, &new_best_length, &new_best_understood, t, about, input_frame, simulated, ""); + + mi.driver = MeterDriver::UNKNOWN; + mi.driver_name = DriverName(""); + + // Use the existing mapping from mfct/media/version to driver. + DriverInfo auto_di = pickMeterDriver(&t); + string auto_driver = auto_di.name().str(); + if (auto_driver == "") { - if (dr == MeterDriver::AUTO) continue; - if (dr == MeterDriver::UNKNOWN) continue; - if (!isMeterDriverReasonableForMedia(dr, "", t.dll_type) && - !isMeterDriverReasonableForMedia(dr, "", t.tpl_type)) - { - // Skip this driver since it is not relevant for this media. - continue; - } + auto_driver = toString(auto_di.driver()); + } - string driver_name = toString(dr); - debug("Testing driver %s...\n", driver_name.c_str()); - mi.driver = dr; - auto meter = createMeter(&mi); - bool match = false; - string id; - bool h = meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); - if (!match) + // Will be non-empty if an explicit driver has been selected. + string force_driver = analyze_driver_; + int force_length = 0; + int force_understood = 0; + + if (force_driver == "" && auto_driver == "") + { + force_driver = auto_driver; + } + + if (force_driver != "") + { + using_driver = findBestOldStyleDriver(mi, &force_length, &force_understood, t, about, input_frame, simulated, + force_driver); + + if (using_driver != "") { - } - else if (!h) - { - // Oups, we added a new meter object tailored for this telegram - // but it still did not handle it! This can happen if the wrong - // decryption key was used. But it is ok if analyzing.... - debug("(meter) newly created meter (%s %s %s) did not handle telegram!\n", - meter->name().c_str(), meter->idsc().c_str(), meter->driverName().str().c_str()); + mi.driver = toMeterDriver(using_driver); + mi.driver_name = DriverName(""); } else { - handled = true; - int l = 0; - int u = 0; - t.analyzeParse(OutputFormat::NONE, &l, &u); - verbose("(analyze) %s %d/%d\n", driver_name.c_str(), u, l); - if (u > best_understood_content_length) - { - // Understood so many bytes - best_understood_content_length = u; - // Out of this many bytes of content total. - best_content_length = l; - best_driver = dr; - } + using_driver = findBestNewStyleDriver(mi, &force_length, &force_understood, t, about, input_frame, simulated, + force_driver); + mi.driver_name = using_driver; + mi.driver = MeterDriver::UNKNOWN; } + using_length = force_length; + using_understood = force_understood; } - for (DriverInfo dr : all_registered_drivers_list_) - { - string driver_name = toString(dr); - debug("Testing driver %s...\n", driver_name.c_str()); - mi.driver_name = driver_name; - auto meter = createMeter(&mi); - bool match = false; - string id; - bool h = meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); - if (!match) - { - } - else if (!h) + if (old_best_understood > new_best_understood) + { + best_length = old_best_length; + best_understood = old_best_understood; + best_driver = best_old_driver; + if (using_driver == "") { - // Oups, we added a new meter object tailored for this telegram - // but it still did not handle it! This can happen if the wrong - // decryption key was used. But it is ok if analyzing.... - debug("(meter) newly created meter (%s %s %s) did not handle telegram!\n", - meter->name().c_str(), meter->idsc().c_str(), meter->driverName().str().c_str()); - } - else - { - handled = true; - int l = 0; - int u = 0; - t.analyzeParse(OutputFormat::NONE, &l, &u); - verbose("(analyze) %s %d/%d\n", driver_name.c_str(), u, l); - if (u > best_understood_content_length) - { - // Understood so many bytes - best_understood_content_length = u; - // Out of this many bytes of content total. - best_content_length = l; - best_driver = dr; - } + mi.driver = toMeterDriver(best_old_driver); + mi.driver_name = DriverName(""); + using_driver = best_old_driver; + using_length = best_length; + using_understood = best_understood; } } - if (handled) + else if (new_best_understood > old_best_understood) { - DriverInfo auto_driver = pickMeterDriver(&t); - string ad = toString(auto_driver); - string bd = toString(best_driver); - if (auto_driver.driver() != MeterDriver::UNKNOWN) + best_length = new_best_length; + best_understood = new_best_understood; + best_driver = best_new_driver; + if (using_driver == "") { - if (ad != bd) - { - printf("\nUsing driver \"%s\" based on mfct/type/version driver lookup table.\n", ad.c_str()); - printf("But a better match could perhaps be driver \"%s\" with %d/%d content bytes understood.\n", - bd.c_str(), best_understood_content_length, best_content_length); - mi.driver_name = auto_driver.name(); - } - else - { - printf("\nUsing driver \"%s\" based on mfct/type/version driver lookup table.\n", ad.c_str()); - printf("Which is also the best matching driver with %d/%d content bytes understood.\n", - best_understood_content_length, best_content_length); - mi.driver_name = best_driver.name(); - } - } - else - { - printf("\nUsing driver \"%s\" based on best content match with %d/%d content bytes understood.\n", - bd.c_str(), best_understood_content_length, best_content_length); - printf("The mfct/type/version combo was not found in the driver lookup table.\n"); - mi.driver_name = best_driver.name(); + mi.driver_name = best_new_driver; + mi.driver = MeterDriver::UNKNOWN; + using_driver = best_new_driver; + using_length = best_length; + using_understood = best_understood; } + } - auto meter = createMeter(&mi); - bool match = false; - string id; - meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); - int l = 0; - int u = 0; - t.analyzeParse(analyze_format_, &l, &u); - string hr, fields, json; - vector envs, more_json, selected_fields; - meter->printMeter(&t, &hr, &fields, '\t', &json, - &envs, &more_json, &selected_fields, true); - printf("%s\n", json.c_str()); - } - else + auto meter = createMeter(&mi); + + bool match = false; + string id; + + meter->handleTelegram(about, input_frame, simulated, &id, &match, &t); + + int u = 0; + int l = 0; + + string output = t.analyzeParse(analyze_format_, &u, &l); + + string hr, fields, json; + vector envs, more_json, selected_fields; + + meter->printMeter(&t, &hr, &fields, '\t', &json, + &envs, &more_json, &selected_fields, true); + + if (auto_driver == "") { - printf("No suitable driver found.\n"); + auto_driver = "not found!"; } + + printf("Auto driver : %s\n", auto_driver.c_str()); + printf("Best driver : %s %02d/%02d\n", best_driver.c_str(), best_understood, best_length); + printf("Using driver : %s %02d/%02d\n", using_driver.c_str(), using_understood, using_length); + + printf("%s\n", output.c_str()); + + printf("%s\n", json.c_str()); } MeterManagerImplementation(bool daemon) : is_daemon_(daemon) {} @@ -809,6 +915,44 @@ void MeterCommonImplementation::addFieldWithExtractor( )); } +void MeterCommonImplementation::addField( + string vname, + Quantity vquantity, + int print_properties, + string help, + function setValueFunc, + function getValueFunc) +{ + string default_unit = unitToStringLowerCase(defaultUnitForQuantity(vquantity)); + string field_name = vname+"_"+default_unit; + fields_.push_back(field_name); + + prints_.push_back( + FieldInfo(vname, + vquantity, + defaultUnitForQuantity(vquantity), + DifVifKey(""), + VifScaling::None, + MeasurementType::Unknown, + ValueInformation::Volume, + StorageNr(0), + TariffNr(0), + 0, + help, + (print_properties & PrintProperty::FIELD) != 0, + (print_properties & PrintProperty::JSON) != 0, + (print_properties & PrintProperty::IMPORTANT) != 0, + field_name, + getValueFunc, + NULL, + setValueFunc, + NULL, + NULL, + NULL, + NoLookup + )); +} + void MeterCommonImplementation::addStringFieldWithExtractor( string vname, Quantity vquantity, @@ -1442,6 +1586,7 @@ bool MeterCommonImplementation::handleTelegram(AboutTelegram &about, vector)> cb) = 0; virtual void whenMeterUpdated(std::function cb) = 0; virtual void pollMeters(shared_ptr bus) = 0; - virtual void analyzeEnabled(bool b, OutputFormat f) = 0; + virtual void analyzeEnabled(bool b, OutputFormat f, string force_driver, string key, bool verbose) = 0; virtual void analyzeTelegram(AboutTelegram &about, vector &input_frame, bool simulated) = 0; virtual ~MeterManager() = default; diff --git a/src/meters_common_implementation.h b/src/meters_common_implementation.h index 1285ed3..2811791 100644 --- a/src/meters_common_implementation.h +++ b/src/meters_common_implementation.h @@ -106,6 +106,14 @@ protected: function setValueFunc, // Use the SET macro above. function getValueFunc); // Use the GET macro above. + void addField( + string vname, // Name of value without unit, eg total + Quantity vquantity, // Value belongs to this quantity. + int print_properties, // Should this be printed by default in fields,json and hr. + string help, + function setValueFunc, // Use the SET macro above. + function getValueFunc); // Use the GET macro above. + #define SET_STRING_FUNC(varname) {[=](string s){varname = s;}} #define GET_STRING_FUNC(varname) {[=](){return varname; }} diff --git a/src/wmbus.cc b/src/wmbus.cc index c23cb73..94becd6 100644 --- a/src/wmbus.cc +++ b/src/wmbus.cc @@ -1,5 +1,5 @@ /* - Copyright (C) 2017-2021 Fredrik Öhrström + Copyright (C) 2017-2022 Fredrik Öhrström This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -808,7 +808,7 @@ void Telegram::addMoreExplanation(int pos, const char* fmt, ...) } } -void Telegram::addSpecialExplanation(int offset, KindOfData k, Understanding u, const char* fmt, ...) +void Telegram::addSpecialExplanation(int offset, int len, KindOfData k, Understanding u, const char* fmt, ...) { char buf[1024]; buf[1023] = 0; @@ -818,7 +818,7 @@ void Telegram::addSpecialExplanation(int offset, KindOfData k, Understanding u, vsnprintf(buf, 1023, fmt, args); va_end(args); - explanations.push_back({offset, 1, buf, k, u}); + explanations.push_back({offset, len, buf, k, u}); } bool expectedMore(int line) @@ -1044,18 +1044,28 @@ bool Telegram::parseELL(vector::iterator &pos) int len = distance(pos+2, frame.end()); uint16_t check = crc16_EN13757(&(frame[dist]), len); - addExplanationAndIncrementPos(pos, 2, KindOfData::PROTOCOL, Understanding::FULL, "%02x%02x payload crc (calculated %02x%02x %s)", - ell_pl_crc_b[0], ell_pl_crc_b[1], - check & 0xff, check >> 8, (ell_pl_crc==check?"OK":"ERROR")); - - if (ell_pl_crc != check && !FUZZING) + if (ell_pl_crc == check || FUZZING) + { + addExplanationAndIncrementPos(pos, 2, KindOfData::PROTOCOL, Understanding::FULL, + "%02x%02x payload crc (calculated %02x%02x %s)", + ell_pl_crc_b[0], ell_pl_crc_b[1], + check & 0xff, check >> 8, (ell_pl_crc==check?"OK":"ERROR")); + } + else { // Ouch, checksum of the payload does not match. - // A wrong key was probably used for decryption. + // A wrong key, or no key was probably used for decryption. decryption_failed = true; + + // Log the content as encrypted. + int num_encrypted_bytes = frame.end()-pos; + string info = bin2hex(pos, frame.end(), num_encrypted_bytes); + info += " encrypted"; + addExplanationAndIncrementPos(pos, num_encrypted_bytes, KindOfData::CONTENT, Understanding::ENCRYPTED, info.c_str()); + if (parser_warns_) { - if (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a)) + if (!beingAnalyzed() && (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a))) { // Print this warning only once! Unless you are using verbose or debug. warning("(wmbus) WARNING! decrypted payload crc failed check, did you use the correct decryption key? " @@ -1493,8 +1503,21 @@ bool Telegram::potentiallyDecrypt(vector::iterator &pos) { addDefaultManufacturerKeyIfAny(frame, tpl_sec_mode, meter_keys); } - bool ok = decrypt_TPL_AES_CBC_IV(this, frame, pos, meter_keys->confidentiality_key); - if (!ok) return false; + int num_encrypted_bytes = 0; + int num_not_encrypted_at_end = 0; + bool ok = decrypt_TPL_AES_CBC_IV(this, frame, pos, meter_keys->confidentiality_key, + &num_encrypted_bytes, &num_not_encrypted_at_end); + if (!ok) + { + string info = bin2hex(pos, frame.end(), num_encrypted_bytes); + info += " encrypted"; + addExplanationAndIncrementPos(pos, num_encrypted_bytes, KindOfData::CONTENT, Understanding::ENCRYPTED, info.c_str()); + if (meter_keys->confidentiality_key.size() > 0) + { + // Only fail if we gave an explicit key. + return false; + } + } // Now the frame from pos and onwards has been decrypted. CHECK(2); @@ -1502,7 +1525,7 @@ bool Telegram::potentiallyDecrypt(vector::iterator &pos) { if (parser_warns_) { - if (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a)) + if (!beingAnalyzed() && (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a))) { // Print this warning only once! Unless you are using verbose or debug. warning("(wmbus) WARNING! decrypted content failed check, did you use the correct decryption key? " @@ -1537,7 +1560,7 @@ bool Telegram::potentiallyDecrypt(vector::iterator &pos) { if (parser_warns_) { - if (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a)) + if (!beingAnalyzed() && (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a))) { // Print this warning only once! Unless you are using verbose or debug. warning("(wmbus) WARNING! telegram mac check failed, did you use the correct decryption key? " @@ -1553,8 +1576,17 @@ bool Telegram::potentiallyDecrypt(vector::iterator &pos) return false; } - bool ok = decrypt_TPL_AES_CBC_NO_IV(this, frame, pos, tpl_generated_key); - if (!ok) return false; + int num_encrypted_bytes = 0; + int num_not_encrypted_at_end = 0; + bool ok = decrypt_TPL_AES_CBC_NO_IV(this, frame, pos, tpl_generated_key, + &num_encrypted_bytes, + &num_not_encrypted_at_end); + if (!ok) + { + addExplanationAndIncrementPos(pos, num_encrypted_bytes, KindOfData::CONTENT, Understanding::FULL, + "encrypted data"); + return false; + } // Now the frame from pos and onwards has been decrypted. CHECK(2); @@ -1562,7 +1594,7 @@ bool Telegram::potentiallyDecrypt(vector::iterator &pos) { if (parser_warns_) { - if (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a)) + if (!beingAnalyzed() && (isVerboseEnabled() || isDebugEnabled() || !warned_for_telegram_before(this, dll_a))) { // Print this warning only once! Unless you are using verbose or debug. warning("(wmbus) WARNING! decrypted content failed check, did you use the correct decryption key? " @@ -1749,12 +1781,22 @@ bool Telegram::parseTPL(vector::iterator &pos) { header_size = distance(frame.begin(), pos); suffix_size = 0; + int num_mfct_bytes = frame.end()-pos; + string info = bin2hex(pos, frame.end(), num_mfct_bytes); + info += " mfct specific"; + addExplanationAndIncrementPos(pos, num_mfct_bytes, KindOfData::CONTENT, Understanding::NONE, info.c_str()); + return true; // Manufacturer specific telegram payload. Oh well.... } case CI_Field_Values::MFCT_SPECIFIC_A3: // Another stoopid non-conformat wmbus Diehl/Sappel/Izar water meter addon. { header_size = distance(frame.begin(), pos); suffix_size = 0; + int num_mfct_bytes = frame.end()-pos; + string info = bin2hex(pos, frame.end(), num_mfct_bytes); + info += " mfct specific"; + addExplanationAndIncrementPos(pos, num_mfct_bytes, KindOfData::CONTENT, Understanding::NONE, info.c_str()); + return true; // Manufacturer specific telegram payload. Oh well.... } } @@ -1989,6 +2031,7 @@ void Telegram::explainParse(string intro, int from) const char *u = "?"; if (p.understanding == Understanding::FULL) u = "!"; if (p.understanding == Understanding::PARTIAL) u = "p"; + if (p.understanding == Understanding::ENCRYPTED) u = "E"; // Do not print ok for understood protocol, it is implicit. // However if a protocol is not full understood then print p or ?. @@ -1998,8 +2041,10 @@ void Telegram::explainParse(string intro, int from) } } -void printAnalysisAsText(vector &explanations, bool use_ansi) +string renderAnalysisAsText(vector &explanations, bool use_ansi) { + string s; + const char *green; const char *yellow; const char *red; @@ -2027,6 +2072,7 @@ void printAnalysisAsText(vector &explanations, bool use_ansi) const char *u = "?"; if (p.understanding == Understanding::FULL) u = "!"; if (p.understanding == Understanding::PARTIAL) u = "p"; + if (p.understanding == Understanding::ENCRYPTED) u = "E"; // Do not print ok for understood protocol, it is implicit. // However if a protocol is not full understood then print p or ?. @@ -2053,16 +2099,17 @@ void printAnalysisAsText(vector &explanations, bool use_ansi) pre = red; } - printf("%03d %s%s: %s%s%s\n", p.pos, c, u, pre, p.info.c_str(), post); + s += tostrprintf("%03d %s%s: %s%s%s\n", p.pos, c, u, pre, p.info.c_str(), post); } + return s; } -void printAnalysisAsJson(vector &explanations) +string renderAnalysisAsJson(vector &explanations) { - printf("{ \"TODO\": true }\n"); + return "{ \"TODO\": true }\n"; } -void Telegram::analyzeParse(OutputFormat format, int *content_length, int *understood_content_length) +string Telegram::analyzeParse(OutputFormat format, int *content_length, int *understood_content_length) { int u = 0; int l = 0; @@ -2073,7 +2120,8 @@ void Telegram::analyzeParse(OutputFormat format, int *content_length, int *under if (e.kind == KindOfData::CONTENT) { l += e.len; - if (e.understanding != Understanding::NONE) + if (e.understanding == Understanding::PARTIAL || + e.understanding == Understanding::FULL) { // Its content and we have at least some understanding. u += e.len; @@ -2089,16 +2137,18 @@ void Telegram::analyzeParse(OutputFormat format, int *content_length, int *under case OutputFormat::TERMINAL: { bool use_ansi = format == OutputFormat::TERMINAL; - printAnalysisAsText(explanations, use_ansi); + return renderAnalysisAsText(explanations, use_ansi); break; } case OutputFormat::JSON: - printAnalysisAsJson(explanations); + return renderAnalysisAsJson(explanations); break; case OutputFormat::NONE: // Do nothing + return ""; break; } + return "ERROR"; } void detectMeterDrivers(int manufacturer, int media, int version, std::vector *drivers); diff --git a/src/wmbus.h b/src/wmbus.h index 6cea13c..da4e56a 100644 --- a/src/wmbus.h +++ b/src/wmbus.h @@ -364,7 +364,7 @@ enum class KindOfData // been partially decoded, or FULL when the volume or energy field is by itself complete. enum class Understanding { - NONE, PARTIAL, FULL + NONE, ENCRYPTED, PARTIAL, FULL }; struct Explanation @@ -514,9 +514,9 @@ struct Telegram void addMoreExplanation(int pos, const char* fmt, ...); void addMoreExplanation(int pos, string json); // Add an explanation of data inside manufacturer specific data. - void addSpecialExplanation(int offset, KindOfData k, Understanding u, const char* fmt, ...); + void addSpecialExplanation(int offset, int len, KindOfData k, Understanding u, const char* fmt, ...); void explainParse(string intro, int from); - void analyzeParse(OutputFormat o, int *content_length, int *understood_content_length); + string analyzeParse(OutputFormat o, int *content_length, int *understood_content_length); bool parserWarns() { return parser_warns_; } bool isSimulated() { return is_simulated_; } diff --git a/src/wmbus_utils.cc b/src/wmbus_utils.cc index 2cce731..182278b 100644 --- a/src/wmbus_utils.cc +++ b/src/wmbus_utils.cc @@ -30,7 +30,6 @@ bool decrypt_ELL_AES_CTR(Telegram *t, vector &frame, vector::itera vector encrypted_bytes; vector decrypted_bytes; encrypted_bytes.insert(encrypted_bytes.end(), pos, frame.end()); - frame.erase(pos, frame.end()); debugPayload("(ELL) decrypting", encrypted_bytes); uchar iv[16]; @@ -82,7 +81,12 @@ bool decrypt_ELL_AES_CTR(Telegram *t, vector &frame, vector::itera incrementIV(iv, sizeof(iv)); } debugPayload("(ELL) decrypted", decrypted_bytes); + + // Remove the encrypted bytes. + frame.erase(pos, frame.end()); + // Insert the decrypted bytes. frame.insert(frame.end(), decrypted_bytes.begin(), decrypted_bytes.end()); + return true; } @@ -92,25 +96,33 @@ string frameTypeKamstrupC1(int ft) { return "?"; } -bool decrypt_TPL_AES_CBC_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey) +bool decrypt_TPL_AES_CBC_IV(Telegram *t, + vector &frame, + vector::iterator &pos, + vector &aeskey, + int *num_encrypted_bytes, + int *num_not_encrypted_at_end) { - if (aeskey.size() == 0) return true; - vector buffer; buffer.insert(buffer.end(), pos, frame.end()); - frame.erase(pos, frame.end()); - debugPayload("(TPL) AES CBC IV decrypting", buffer); - size_t len = buffer.size(); + size_t len = frame.end()-pos; if (t->tpl_num_encr_blocks) { len = t->tpl_num_encr_blocks*16; } + *num_encrypted_bytes = len; + *num_not_encrypted_at_end = buffer.size()-len; + debug("(TPL) num encrypted blocks %zu (%d bytes and remaining unencrypted %zu bytes)\n", t->tpl_num_encr_blocks, len, buffer.size()-len); + if (aeskey.size() == 0) return false; + + debugPayload("(TPL) AES CBC IV decrypting", buffer); + if (buffer.size() < len) { warning("(TPL) warning: decryption received less bytes than expected for decryption! " @@ -162,7 +174,11 @@ bool decrypt_TPL_AES_CBC_IV(Telegram *t, vector &frame, vector::it uchar decrypted_data[buffer.size()]; AES_CBC_decrypt_buffer(decrypted_data, buffer_data, len, &aeskey[0], iv); + // Remove the encrypted bytes. + frame.erase(pos, frame.end()); + // Insert the decrypted bytes. frame.insert(frame.end(), decrypted_data, decrypted_data+len); + debugPayload("(TPL) decrypted ", frame, pos); if (len < buffer.size()) @@ -173,7 +189,9 @@ bool decrypt_TPL_AES_CBC_IV(Telegram *t, vector &frame, vector::it return true; } -bool decrypt_TPL_AES_CBC_NO_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey) +bool decrypt_TPL_AES_CBC_NO_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey, + int *num_encrypted_bytes, + int *num_not_encrypted_at_end) { if (aeskey.size() == 0) return true; diff --git a/src/wmbus_utils.h b/src/wmbus_utils.h index 297f4c0..26da698 100644 --- a/src/wmbus_utils.h +++ b/src/wmbus_utils.h @@ -23,8 +23,13 @@ #include "wmbus.h" bool decrypt_ELL_AES_CTR(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey); -bool decrypt_TPL_AES_CBC_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey); -bool decrypt_TPL_AES_CBC_NO_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey); +bool decrypt_TPL_AES_CBC_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey, + int *num_encrypted_bytes, + int *num_not_encrypted_at_end); +bool decrypt_TPL_AES_CBC_NO_IV(Telegram *t, vector &frame, vector::iterator &pos, vector &aeskey, + int *num_encrypted_bytes, + int *num_not_encrypted_at_end); + string frameTypeKamstrupC1(int ft); #endif diff --git a/test.sh b/test.sh index a9cde1a..25e4409 100755 --- a/test.sh +++ b/test.sh @@ -145,8 +145,8 @@ if [ "$?" != "0" ]; then RC="1"; fi ./tests/test_drivers.sh $PROG if [ "$?" != "0" ]; then RC="1"; fi -#./tests/test_analyze.sh $PROG -#if [ "$?" != "0" ]; then RC="1"; fi +./tests/test_analyze.sh $PROG +if [ "$?" != "0" ]; then RC="1"; fi if [ -x ../additional_tests.sh ] then diff --git a/tests/test_analyze.sh b/tests/test_analyze.sh index 372b321..f61a48d 100755 --- a/tests/test_analyze.sh +++ b/tests/test_analyze.sh @@ -4,13 +4,72 @@ PROG="$1" TEST=testoutput mkdir -p $TEST -TESTNAME="Test analyze compact telegram" +performCheck() { +if [ "$?" = "0" ] +then + cat $TEST/test_output.txt | sed 's/"timestamp":"....-..-..T..:..:..Z"/"timestamp":"1111-11-11T11:11:11Z"/' > $TEST/test_response.txt + diff $TEST/test_expected.txt $TEST/test_response.txt + if [ "$?" = "0" ] + then + echo "OK: $TESTNAME" + TESTRESULT="OK" + else + echo "ERROR: $TESTNAME $0" + fi +else + echo "ERROR: $TESTNAME $0" + echo "wmbusmeters returned error code: $?" + cat $TEST/test_output.txt +fi +} + +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +TESTNAME="Test analyze encrypted (no-key) ctr full telegram" TESTRESULT="ERROR" cat > $TEST/test_expected.txt < $TEST/test_output.txt 2>&1 + +performCheck + +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +TESTNAME="Test analyze encrypted (no-key) ctr compact telegram" +TESTRESULT="ERROR" + +cat > $TEST/test_expected.txt < $TEST/test_output.txt 1>&2 +$PROG --analyze 23442D2C998734761B168D20983081B2227A6FA1F10E1B79B5EB4B17E81F930E937EE06C > $TEST/test_output.txt 2>&1 -if [ "$?" = "0" ] -then - diff $TEST/test_expected.txt $TEST/test_output.txt - if [ "$?" = "0" ] - then - echo OK: $TESTNAME - TESTRESULT="OK" - fi -else - echo ERROR: $TESTNAME - echo "wmbusmeters returned error code: $?" - cat $TEST/test_output.txt -fi +performCheck -TESTNAME="Test analyze normal telegram" +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +TESTNAME="Test analyze encrypted (with-key) ctr full telegram" TESTRESULT="ERROR" cat > $TEST/test_expected.txt < $TEST/test_output.txt 1>&2 +$PROG --analyze=28F64A24988064A079AA2C807D6102AE 2A442D2C998734761B168D2091D37CAC21E1D68CDAFFCD3DC452BD802913FF7B1706CA9E355D6C2701CC24 > $TEST/test_output.txt 2>&1 -if [ "$?" = "0" ] -then - diff $TEST/test_expected.txt $TEST/test_output.txt - if [ "$?" = "0" ] - then - echo OK: $TESTNAME - TESTRESULT="OK" - fi -else - echo "wmbusmeters returned error code: $?" - cat $TEST/test_output.txt -fi +performCheck + +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +TESTNAME="Test analyze encrypted (no-key) ctr compact telegram" +TESTRESULT="ERROR" + +cat > $TEST/test_expected.txt < $TEST/test_output.txt 2>&1 + +performCheck + +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +TESTNAME="Test analyze CBC IV (no-key)" +TESTRESULT="ERROR" + +cat > $TEST/test_expected.txt < $TEST/test_output.txt 2>&1 + +performCheck + +######################################################################################################################## +######################################################################################################################## +######################################################################################################################## + +#TESTNAME="Test analyze CBC IV (with-key)" +#TESTRESULT="ERROR" + +#cat > $TEST/test_expected.txt < $TEST/test_output.txt 2>&1 + +#performCheck