From 20bc6c6a022a918224e1de614a142a7ba3cbf86d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fredrik=20=C3=96hrstr=C3=B6m?= <oehrstroem@gmail.com>
Date: Sat, 9 May 2020 23:43:30 +0200
Subject: [PATCH] Add --selectfields=...

---
 CHANGES                            | 13 +++++
 README.md                          | 16 ++++--
 src/cmdline.cc                     | 19 ++++++-
 src/config.cc                      | 13 +++++
 src/config.h                       |  3 ++
 src/main.cc                        | 27 ++++++++--
 src/meters.cc                      | 81 ++++++++++++++++++++++++++++--
 src/meters.h                       |  6 ++-
 src/meters_common_implementation.h |  5 +-
 src/printer.cc                     |  6 ++-
 src/printer.h                      |  2 +-
 test.sh                            |  3 ++
 tests/test_additional_json.sh      |  2 +-
 tests/test_fields.sh               | 37 ++++++++++++++
 wmbusmeters.1                      |  2 +-
 15 files changed, 217 insertions(+), 18 deletions(-)
 create mode 100755 tests/test_fields.sh

diff --git a/CHANGES b/CHANGES
index 18444be..1841e96 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,16 @@
+Version 0.9.31: 2020-05-09
+
+You can add comments in the conf and meter files.
+A comment is a line that starts with #.
+
+When using --format=fields you can now supply
+--selectfields=id,total_m3 to print only the selected fields.
+Run with --listfields to see all the avilable fields from a meter.
+
+Improvements in the izar driver. Thanks Erwan!
+
+Bug fixes in the amd8465 driver. Thanks Henry N.!
+
 Version 0.9.30: 2020-04-22
 
 Erwan added support for building on FreeBSD. Thanks Erwan!
diff --git a/README.md b/README.md
index dceeb88..665a03d 100644
--- a/README.md
+++ b/README.md
@@ -133,6 +133,8 @@ As <options> you can use:
     --listento=c1,t1,s1 tell the wmbus dongle to listen to these link modes
                       different dongles support different combinations of modes
     --c1 --t1 --s1 --s1m ... another way to set the link mode for the dongle
+    --listenvs list the env variables available for the meter
+    --listfields list the fields selectable for the meter
     --logfile=<file> use this file instead of stdout
     --logtelegrams log the contents of the telegrams for easy replay
     --meterfiles=<dir> store meter readings in dir
@@ -142,9 +144,9 @@ As <options> you can use:
                           timestamp (localtime) with the given resolution.
     --oneshot wait for an update from each meter, then quit
     --reopenafter=<time> close/reopen dongle connection repeatedly every <time> seconds, eg 60s, 60m, 24h
+    --selectfields=id,timestamp,total_m3 select fields to be printed
     --separator=<c> change field separator to c
     --shell=<cmdline> invokes cmdline with env variables containing the latest reading
-    --shellenvs list the env variables available for the meter
     --useconfig=<dir> load config files from dir/etc
     --usestderr write debug/verbose and logging output to stderr
     --verbose for more information
@@ -279,6 +281,14 @@ default 868.95MHz.
 
 `GreenhouseTapWater;33333333;9999.099;77.712;0.000;11;31;;2018-03-05 12:10.24`
 
+You can select a subset of all available fields:
+
+`wmbusmeters --format=fields --selectfields=id,total_m3 /dev/ttyUSB0:im871a GreenhouseWater multical21 33333333 NOKEY`
+
+`33333333;9999.099`
+
+You can list all available fields for the meter by adding `--listfields` to the command line.
+
 Eaxmple of using the shell command to publish to MQTT:
 
 `wmbusmeters --shell='HOME=/home/you mosquitto_pub -h localhost -t water -m "$METER_JSON"' /dev/ttyUSB0:im871a GreenhouseWater multical21 33333333 NOKEY`
@@ -289,8 +299,8 @@ Eaxmple of using the shell command to inject data into postgresql database:
 
 You can have multiple shell commands and they will be executed in the order you gave them on the commandline.
 Note that to single quotes around the command is necessary to pass the env variable names into wmbusmeters.
-To list the shell env variables available for your meter, add --shellenvs to the commandline:
-`wmbusmeters --shellenvs /dev/ttyUSB1:cul Water iperl 12345678 NOKEY`
+To list the shell env variables available for your meter, add --listenvs to the commandline:
+`wmbusmeters --listenvs /dev/ttyUSB1:cul Water iperl 12345678 NOKEY`
 which outputs:
 ```
 Environment variables provided to shell for meter iperl:
diff --git a/src/cmdline.cc b/src/cmdline.cc
index 0c073cb..c8cb8d2 100644
--- a/src/cmdline.cc
+++ b/src/cmdline.cc
@@ -217,6 +217,18 @@ unique_ptr<Configuration> parseCommandLine(int argc, char **argv) {
             i++;
             continue;
         }
+        if (!strncmp(argv[i], "--selectfields=", 15)) {
+            if (strlen(argv[i]) > 15)
+            {
+                string s = string(argv[i]+15);
+                handleSelectedFields(c, s);
+            } else {
+                error("You must supply fields to be selected.\n");
+            }
+            i++;
+            continue;
+        }
+
         if (!strncmp(argv[i], "--separator=", 12)) {
             if (!c->fields) {
                 error("You must specify --format=fields before --separator=X\n");
@@ -355,11 +367,16 @@ unique_ptr<Configuration> parseCommandLine(int argc, char **argv) {
             i++;
             continue;
         }
-        if (!strncmp(argv[i], "--shellenvs", 11)) {
+        if (!strncmp(argv[i], "--listenvs", 10)) {
             c->list_shell_envs = true;
             i++;
             continue;
         }
+        if (!strncmp(argv[i], "--listfields", 12)) {
+            c->list_fields = true;
+            i++;
+            continue;
+        }
         if (!strcmp(argv[i], "--oneshot")) {
             c->oneshot = true;
             i++;
diff --git a/src/config.cc b/src/config.cc
index b86323f..8f7455a 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -342,6 +342,18 @@ void handleConversions(Configuration *c, string s)
     }
 }
 
+void handleSelectedFields(Configuration *c, string s)
+{
+    char buf[s.length()+1];
+    strcpy(buf, s.c_str());
+    const char *tok = strtok(buf, ",");
+    while (tok != NULL)
+    {
+        c->selected_fields.push_back(tok);
+        tok = strtok(NULL, ",");
+    }
+}
+
 void handleShell(Configuration *c, string cmdline)
 {
     c->shells.push_back(cmdline);
@@ -388,6 +400,7 @@ unique_ptr<Configuration> loadConfiguration(string root, string device_override,
         else if (p.first == "reopenafter") handleReopenAfter(c, p.second);
         else if (p.first == "separator") handleSeparator(c, p.second);
         else if (p.first == "addconversions") handleConversions(c, p.second);
+        else if (p.first == "selectfields") handleSelectedFields(c, p.second);
         else if (p.first == "shell") handleShell(c, p.second);
         else if (startsWith(p.first, "json_"))
         {
diff --git a/src/config.h b/src/config.h
index b8f4cdf..5b7eeb9 100644
--- a/src/config.h
+++ b/src/config.h
@@ -70,6 +70,7 @@ struct Configuration
     char separator { ';' };
     std::vector<std::string> shells;
     bool list_shell_envs {};
+    bool list_fields {};
     bool oneshot {};
     int  exitafter {}; // Seconds to exit.
     int  reopenafter {}; // Re-open the serial device repeatedly. Silly dongle.
@@ -81,6 +82,7 @@ struct Configuration
     bool link_mode_configured {};
     bool no_init {};
     std::vector<Unit> conversions;
+    std::vector<std::string> selected_fields;
     std::vector<MeterInfo> meters;
     std::vector<std::string> jsons; // Additional jsons to always add.
 
@@ -90,6 +92,7 @@ struct Configuration
 unique_ptr<Configuration> loadConfiguration(string root, string device_override, string listento_override);
 
 void handleConversions(Configuration *c, string s);
+void handleSelectedFields(Configuration *c, string s);
 
 enum class LinkModeCalculationResultType
 {
diff --git a/src/main.cc b/src/main.cc
index 871b818..1895baf 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -302,7 +302,8 @@ LIST_OF_METERS
                                           &ignore2, config->separator,
                                           &ignore3,
                                           &envs,
-                                          &config->jsons);
+                                          &config->jsons,
+                                          &config->selected_fields);
                 printf("Environment variables provided to shell for meter %s:\n", m.type.c_str());
                 for (auto &e : envs) {
                     int p = e.find('=');
@@ -311,8 +312,28 @@ LIST_OF_METERS
                 }
                 exit(0);
             }
-            meters.back()->onUpdate([&](Telegram*t,Meter* meter) { output->print(t,meter,&config->jsons); });
-            meters.back()->onUpdate([&](Telegram*t, Meter* meter) { oneshotCheck(config, manager.get(), t, meter, meters); });
+
+            if (config->list_fields)
+            {
+                printf("Fields produced by meter %s:\n", m.type.c_str());
+                printf("id\n");
+                printf("name\n");
+                printf("timestamp\n");
+                for (auto &f : meters.back()->fields())
+                {
+                    printf("%s\n", f.c_str());
+                }
+                exit(0);
+            }
+
+            meters.back()->onUpdate([&](Telegram*t,Meter* meter)
+                                    {
+                                        output->print(t,meter,&config->jsons,&config->selected_fields);
+                                    });
+            meters.back()->onUpdate([&](Telegram*t, Meter* meter)
+                                    {
+                                        oneshotCheck(config, manager.get(), t, meter, meters);
+                                    });
         }
     }
     else
diff --git a/src/meters.cc b/src/meters.cc
index 30672f8..c596e5a 100755
--- a/src/meters.cc
+++ b/src/meters.cc
@@ -101,6 +101,8 @@ void MeterCommonImplementation::addLinkMode(LinkMode lm)
 void MeterCommonImplementation::addPrint(string vname, Quantity vquantity,
                                          function<double(Unit)> getValueFunc, string help, bool field, bool json)
 {
+    string default_unit = unitToStringLowerCase(defaultUnitForQuantity(vquantity));
+    fields_.push_back(vname+"_"+default_unit);
     prints_.push_back( { vname, vquantity, defaultUnitForQuantity(vquantity), getValueFunc, NULL, help, field, json });
 }
 
@@ -127,6 +129,11 @@ vector<string> MeterCommonImplementation::ids()
     return ids_;
 }
 
+vector<string> MeterCommonImplementation::fields()
+{
+    return fields_;
+}
+
 string MeterCommonImplementation::name()
 {
     return name_;
@@ -256,7 +263,7 @@ void MeterCommonImplementation::triggerUpdate(Telegram *t)
     t->handled = true;
 }
 
-string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr)
+string concatAllFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr)
 {
     string s;
     s = "";
@@ -288,6 +295,71 @@ string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector
     return s;
 }
 
+string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr,
+                    vector<string> *selected_fields)
+{
+    if (selected_fields == NULL || selected_fields->size() == 0)
+    {
+        return concatAllFields(m, t, c, prints, cs, hr);
+    }
+    string s;
+    s = "";
+
+    for (string field : *selected_fields)
+    {
+        if (field == "name")
+        {
+            s += m->name() + c;
+            continue;
+        }
+        if (field == "id")
+        {
+            s += t->id + c;
+            continue;
+        }
+        if (field == "timestamp")
+        {
+            s += m->datetimeOfUpdateHumanReadable() + c;
+            continue;
+        }
+
+        for (Print p : prints)
+        {
+            if (p.getValueString)
+            {
+                if (field == p.vname)
+                {
+                    s += p.getValueString() + c;
+                }
+            }
+            else if (p.getValueDouble)
+            {
+                string default_unit = unitToStringLowerCase(p.default_unit);
+                string var = p.vname+"_"+default_unit;
+                if (field == var)
+                {
+                    s += valueToString(p.getValueDouble(p.default_unit), p.default_unit) + c;
+                }
+                else
+                {
+                    Unit u = replaceWithConversionUnit(p.default_unit, cs);
+                    if (u != p.default_unit)
+                    {
+                        string unit = unitToStringLowerCase(u);
+                        string var = p.vname+"_"+unit;
+                        if (field == var)
+                        {
+                            s += valueToString(p.getValueDouble(u), u) + c;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (s.back() == c) s.pop_back();
+    return s;
+}
+
 bool MeterCommonImplementation::handleTelegram(vector<uchar> input_frame)
 {
     Telegram t;
@@ -340,10 +412,11 @@ void MeterCommonImplementation::printMeter(Telegram *t,
                                            string *fields, char separator,
                                            string *json,
                                            vector<string> *envs,
-                                           vector<string> *more_json)
+                                           vector<string> *more_json,
+                                           vector<string> *selected_fields)
 {
-    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true);
-    *fields = concatFields(this, t, separator, prints_, conversions_, false);
+    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true, selected_fields);
+    *fields = concatFields(this, t, separator, prints_, conversions_, false, selected_fields);
 
     string s;
     s += "{";
diff --git a/src/meters.h b/src/meters.h
index 7e07483..7b8f6ca 100644
--- a/src/meters.h
+++ b/src/meters.h
@@ -88,7 +88,10 @@ struct MeterInfo
 
 struct Meter
 {
+    // This meter listens to these ids.
     virtual vector<string> ids() = 0;
+    // This meter can report these fields, like total_m3, temp_c.
+    virtual vector<string> fields() = 0;
     virtual string meterName() = 0;
     virtual string name() = 0;
     virtual MeterType type() = 0;
@@ -106,7 +109,8 @@ struct Meter
                             string *fields, char separator,
                             string *json,
                             vector<string> *envs,
-                            vector<string> *more_json) = 0;
+                            vector<string> *more_json,
+                            vector<string> *selected_fields) = 0;
 
     // The handleTelegram expects an input_frame where the DLL crcs have been removed.
     bool handleTelegram(vector<uchar> input_frame);
diff --git a/src/meters_common_implementation.h b/src/meters_common_implementation.h
index ffb5a37..ed2770d 100755
--- a/src/meters_common_implementation.h
+++ b/src/meters_common_implementation.h
@@ -39,6 +39,7 @@ struct Print
 struct MeterCommonImplementation : public virtual Meter
 {
     vector<string> ids();
+    vector<string> fields();
     string name();
     MeterType type();
     vector<int> media();
@@ -94,7 +95,8 @@ protected:
                     string *fields, char separator,
                     string *json,
                     vector<string> *envs,
-                    vector<string> *more_json); // Add this json "key"="value" strings.
+                    vector<string> *more_json, // Add this json "key"="value" strings.
+                    vector<string> *selected_fields); // Only print these fields. Json always everything.
 
     virtual void processContent(Telegram *t) = 0;
 
@@ -121,6 +123,7 @@ protected:
     std::map<std::string,std::pair<int,std::string>> values_;
     vector<Unit> conversions_;
     vector<Print> prints_;
+    vector<string> fields_;
 };
 
 #endif
diff --git a/src/printer.cc b/src/printer.cc
index 77a539a..c8eff89 100644
--- a/src/printer.cc
+++ b/src/printer.cc
@@ -40,13 +40,15 @@ Printer::Printer(bool json, bool fields, char separator,
     timestamp_ = timestamp;
 }
 
-void Printer::print(Telegram *t, Meter *meter, vector<string> *more_json)
+void Printer::print(Telegram *t, Meter *meter,
+                    vector<string> *more_json,
+                    vector<string> *selected_fields)
 {
     string human_readable, fields, json;
     vector<string> envs;
     bool printed = false;
 
-    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json);
+    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json, selected_fields);
 
     if (shell_cmdlines_.size() > 0 || meter->shellCmdlines().size() > 0) {
         printShells(meter, envs);
diff --git a/src/printer.h b/src/printer.h
index 63a397b..dff5b48 100644
--- a/src/printer.h
+++ b/src/printer.h
@@ -32,7 +32,7 @@ struct Printer {
             MeterFileNaming naming,
             MeterFileTimestamp timestamp);
 
-    void print(Telegram *t, Meter *meter, vector<string> *more_json);
+    void print(Telegram *t, Meter *meter, vector<string> *more_json, vector<string> *selected_fields);
 
     private:
 
diff --git a/test.sh b/test.sh
index c892763..1cf706a 100755
--- a/test.sh
+++ b/test.sh
@@ -48,6 +48,9 @@ if [ "$?" != "0" ]; then RC="1"; fi
 tests/test_conversions.sh $PROG
 if [ "$?" != "0" ]; then RC="1"; fi
 
+tests/test_fields.sh $PROG
+if [ "$?" != "0" ]; then RC="1"; fi
+
 #tests/test_oneshot.sh $PROG broken test
 #if [ "$?" != "0" ]; then RC="1"; fi
 
diff --git a/tests/test_additional_json.sh b/tests/test_additional_json.sh
index cb92b1f..da68533 100755
--- a/tests/test_additional_json.sh
+++ b/tests/test_additional_json.sh
@@ -29,7 +29,7 @@ if [ "$TESTRESULT" = "ERROR" ]; then echo ERROR: $TESTNAME;  exit 1; fi
 TESTNAME="Test additional shell envs from cmdline"
 TESTRESULT="ERROR"
 
-$PROG --format=json --json_floor=5 --json_house="alfa beta" --shellenvs --listento=c1 simulations/simulation_additional_json.txt \
+$PROG --format=json --json_floor=5 --json_house="alfa beta" --listenvs --listento=c1 simulations/simulation_additional_json.txt \
       Vatten multical21 76348799 "" | grep METER_  > $TEST/test_output.txt
 
 ENVS=$(cat $TEST/test_output.txt | tr '\n' ' ')
diff --git a/tests/test_fields.sh b/tests/test_fields.sh
new file mode 100755
index 0000000..da5b1f2
--- /dev/null
+++ b/tests/test_fields.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+PROG="$1"
+
+mkdir -p testoutput
+
+TEST=testoutput
+
+TESTNAME="Test selected fields"
+TESTRESULT="ERROR"
+
+cat <<EOF > $TEST/test_expected.txt
+76348799;Vatten;6408;6.408;0;127;260.6
+76348799;Vatten;6408;6.408;0;127;260.6
+EOF
+
+$PROG --format=fields --separator=';' \
+      --selectfields=id,name,total_l,total_m3,max_flow_m3h,flow_temperature_c,flow_temperature_f \
+      --addconversions=L,F \
+      simulations/simulation_c1.txt Vatten multical21 76348799 "" \
+      > $TEST/test_output.txt
+
+if [ "$?" = "0" ]
+then
+    diff $TEST/test_expected.txt $TEST/test_output.txt
+    if [ "$?" = "0" ]
+    then
+        echo OK: $TESTNAME
+        TESTRESULT="OK"
+    fi
+fi
+
+if [ "$TESTRESULT" = "ERROR" ]
+then
+    echo ERROR: $TESTNAME
+    exit 1
+fi
diff --git a/wmbusmeters.1 b/wmbusmeters.1
index 0c15c4c..f6d81ea 100644
--- a/wmbusmeters.1
+++ b/wmbusmeters.1
@@ -57,7 +57,7 @@ mqtt_publish) sent to a REST API (eg curl) or store it in a database
 
 \fB\--shell=\fR<cmdline> invokes cmdline with env variables containing the latest reading
 
-\fB\--shellenvs\fR list the env variables available for the meter
+\fB\--listenvs\fR list the env variables available for the meter
 
 \fB\--useconfig=\fR<dir> load config files from dir/etc