Add --selectfields=...

CHANGES

@@ -1,3 +1,16 @@
+Version 0.9.31: 2020-05-09
+
+You can add comments in the conf and meter files.
+A comment is a line that starts with #.
+
+When using --format=fields you can now supply
+--selectfields=id,total_m3 to print only the selected fields.
+Run with --listfields to see all the avilable fields from a meter.
+
+Improvements in the izar driver. Thanks Erwan!
+
+Bug fixes in the amd8465 driver. Thanks Henry N.!
+
 Version 0.9.30: 2020-04-22
 
 Erwan added support for building on FreeBSD. Thanks Erwan!

README.md

@@ -133,6 +133,8 @@ As <options> you can use:
     --listento=c1,t1,s1 tell the wmbus dongle to listen to these link modes
                       different dongles support different combinations of modes
     --c1 --t1 --s1 --s1m ... another way to set the link mode for the dongle
+    --listenvs list the env variables available for the meter
+    --listfields list the fields selectable for the meter
     --logfile=<file> use this file instead of stdout
     --logtelegrams log the contents of the telegrams for easy replay
     --meterfiles=<dir> store meter readings in dir
@@ -142,9 +144,9 @@ As <options> you can use:
                           timestamp (localtime) with the given resolution.
     --oneshot wait for an update from each meter, then quit
     --reopenafter=<time> close/reopen dongle connection repeatedly every <time> seconds, eg 60s, 60m, 24h
+    --selectfields=id,timestamp,total_m3 select fields to be printed
     --separator=<c> change field separator to c
     --shell=<cmdline> invokes cmdline with env variables containing the latest reading
-    --shellenvs list the env variables available for the meter
     --useconfig=<dir> load config files from dir/etc
     --usestderr write debug/verbose and logging output to stderr
     --verbose for more information
@@ -279,6 +281,14 @@ default 868.95MHz.
 
 `GreenhouseTapWater;33333333;9999.099;77.712;0.000;11;31;;2018-03-05 12:10.24`
 
+You can select a subset of all available fields:
+
+`wmbusmeters --format=fields --selectfields=id,total_m3 /dev/ttyUSB0:im871a GreenhouseWater multical21 33333333 NOKEY`
+
+`33333333;9999.099`
+
+You can list all available fields for the meter by adding `--listfields` to the command line.
+
 Eaxmple of using the shell command to publish to MQTT:
 
 `wmbusmeters --shell='HOME=/home/you mosquitto_pub -h localhost -t water -m "$METER_JSON"' /dev/ttyUSB0:im871a GreenhouseWater multical21 33333333 NOKEY`
@@ -289,8 +299,8 @@ Eaxmple of using the shell command to inject data into postgresql database:
 
 You can have multiple shell commands and they will be executed in the order you gave them on the commandline.
 Note that to single quotes around the command is necessary to pass the env variable names into wmbusmeters.
-To list the shell env variables available for your meter, add --shellenvs to the commandline:
-`wmbusmeters --shellenvs /dev/ttyUSB1:cul Water iperl 12345678 NOKEY`
+To list the shell env variables available for your meter, add --listenvs to the commandline:
+`wmbusmeters --listenvs /dev/ttyUSB1:cul Water iperl 12345678 NOKEY`
 which outputs:
 ```
 Environment variables provided to shell for meter iperl:

src/cmdline.cc

@@ -217,6 +217,18 @@ unique_ptr<Configuration> parseCommandLine(int argc, char **argv) {
             i++;
             continue;
         }
+        if (!strncmp(argv[i], "--selectfields=", 15)) {
+            if (strlen(argv[i]) > 15)
+            {
+                string s = string(argv[i]+15);
+                handleSelectedFields(c, s);
+            } else {
+                error("You must supply fields to be selected.\n");
+            }
+            i++;
+            continue;
+        }
+
         if (!strncmp(argv[i], "--separator=", 12)) {
             if (!c->fields) {
                 error("You must specify --format=fields before --separator=X\n");
@@ -355,11 +367,16 @@ unique_ptr<Configuration> parseCommandLine(int argc, char **argv) {
             i++;
             continue;
         }
-        if (!strncmp(argv[i], "--shellenvs", 11)) {
+        if (!strncmp(argv[i], "--listenvs", 10)) {
             c->list_shell_envs = true;
             i++;
             continue;
         }
+        if (!strncmp(argv[i], "--listfields", 12)) {
+            c->list_fields = true;
+            i++;
+            continue;
+        }
         if (!strcmp(argv[i], "--oneshot")) {
             c->oneshot = true;
             i++;

src/config.cc

@@ -342,6 +342,18 @@ void handleConversions(Configuration *c, string s)
     }
 }
 
+void handleSelectedFields(Configuration *c, string s)
+{
+    char buf[s.length()+1];
+    strcpy(buf, s.c_str());
+    const char *tok = strtok(buf, ",");
+    while (tok != NULL)
+    {
+        c->selected_fields.push_back(tok);
+        tok = strtok(NULL, ",");
+    }
+}
+
 void handleShell(Configuration *c, string cmdline)
 {
     c->shells.push_back(cmdline);
@@ -388,6 +400,7 @@ unique_ptr<Configuration> loadConfiguration(string root, string device_override,
         else if (p.first == "reopenafter") handleReopenAfter(c, p.second);
         else if (p.first == "separator") handleSeparator(c, p.second);
         else if (p.first == "addconversions") handleConversions(c, p.second);
+        else if (p.first == "selectfields") handleSelectedFields(c, p.second);
         else if (p.first == "shell") handleShell(c, p.second);
         else if (startsWith(p.first, "json_"))
         {

src/config.h

@@ -70,6 +70,7 @@ struct Configuration
     char separator { ';' };
     std::vector<std::string> shells;
     bool list_shell_envs {};
+    bool list_fields {};
     bool oneshot {};
     int  exitafter {}; // Seconds to exit.
     int  reopenafter {}; // Re-open the serial device repeatedly. Silly dongle.
@@ -81,6 +82,7 @@ struct Configuration
     bool link_mode_configured {};
     bool no_init {};
     std::vector<Unit> conversions;
+    std::vector<std::string> selected_fields;
     std::vector<MeterInfo> meters;
     std::vector<std::string> jsons; // Additional jsons to always add.
 
@@ -90,6 +92,7 @@ struct Configuration
 unique_ptr<Configuration> loadConfiguration(string root, string device_override, string listento_override);
 
 void handleConversions(Configuration *c, string s);
+void handleSelectedFields(Configuration *c, string s);
 
 enum class LinkModeCalculationResultType
 {

src/main.cc

@@ -302,7 +302,8 @@ LIST_OF_METERS
                                           &ignore2, config->separator,
                                           &ignore3,
                                           &envs,
-                                          &config->jsons);
+                                          &config->jsons,
+                                          &config->selected_fields);
                 printf("Environment variables provided to shell for meter %s:\n", m.type.c_str());
                 for (auto &e : envs) {
                     int p = e.find('=');
@@ -311,8 +312,28 @@ LIST_OF_METERS
                 }
                 exit(0);
             }
-            meters.back()->onUpdate([&](Telegram*t,Meter* meter) { output->print(t,meter,&config->jsons); });
-            meters.back()->onUpdate([&](Telegram*t, Meter* meter) { oneshotCheck(config, manager.get(), t, meter, meters); });
+
+            if (config->list_fields)
+            {
+                printf("Fields produced by meter %s:\n", m.type.c_str());
+                printf("id\n");
+                printf("name\n");
+                printf("timestamp\n");
+                for (auto &f : meters.back()->fields())
+                {
+                    printf("%s\n", f.c_str());
+                }
+                exit(0);
+            }
+
+            meters.back()->onUpdate([&](Telegram*t,Meter* meter)
+                                    {
+                                        output->print(t,meter,&config->jsons,&config->selected_fields);
+                                    });
+            meters.back()->onUpdate([&](Telegram*t, Meter* meter)
+                                    {
+                                        oneshotCheck(config, manager.get(), t, meter, meters);
+                                    });
         }
     }
     else

src/meters.cc

@@ -101,6 +101,8 @@ void MeterCommonImplementation::addLinkMode(LinkMode lm)
 void MeterCommonImplementation::addPrint(string vname, Quantity vquantity,
                                          function<double(Unit)> getValueFunc, string help, bool field, bool json)
 {
+    string default_unit = unitToStringLowerCase(defaultUnitForQuantity(vquantity));
+    fields_.push_back(vname+"_"+default_unit);
     prints_.push_back( { vname, vquantity, defaultUnitForQuantity(vquantity), getValueFunc, NULL, help, field, json });
 }
 
@@ -127,6 +129,11 @@ vector<string> MeterCommonImplementation::ids()
     return ids_;
 }
 
+vector<string> MeterCommonImplementation::fields()
+{
+    return fields_;
+}
+
 string MeterCommonImplementation::name()
 {
     return name_;
@@ -256,7 +263,7 @@ void MeterCommonImplementation::triggerUpdate(Telegram *t)
     t->handled = true;
 }
 
-string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr)
+string concatAllFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr)
 {
     string s;
     s = "";
@@ -288,6 +295,71 @@ string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector
     return s;
 }
 
+string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr,
+                    vector<string> *selected_fields)
+{
+    if (selected_fields == NULL || selected_fields->size() == 0)
+    {
+        return concatAllFields(m, t, c, prints, cs, hr);
+    }
+    string s;
+    s = "";
+
+    for (string field : *selected_fields)
+    {
+        if (field == "name")
+        {
+            s += m->name() + c;
+            continue;
+        }
+        if (field == "id")
+        {
+            s += t->id + c;
+            continue;
+        }
+        if (field == "timestamp")
+        {
+            s += m->datetimeOfUpdateHumanReadable() + c;
+            continue;
+        }
+
+        for (Print p : prints)
+        {
+            if (p.getValueString)
+            {
+                if (field == p.vname)
+                {
+                    s += p.getValueString() + c;
+                }
+            }
+            else if (p.getValueDouble)
+            {
+                string default_unit = unitToStringLowerCase(p.default_unit);
+                string var = p.vname+"_"+default_unit;
+                if (field == var)
+                {
+                    s += valueToString(p.getValueDouble(p.default_unit), p.default_unit) + c;
+                }
+                else
+                {
+                    Unit u = replaceWithConversionUnit(p.default_unit, cs);
+                    if (u != p.default_unit)
+                    {
+                        string unit = unitToStringLowerCase(u);
+                        string var = p.vname+"_"+unit;
+                        if (field == var)
+                        {
+                            s += valueToString(p.getValueDouble(u), u) + c;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (s.back() == c) s.pop_back();
+    return s;
+}
+
 bool MeterCommonImplementation::handleTelegram(vector<uchar> input_frame)
 {
     Telegram t;
@@ -340,10 +412,11 @@ void MeterCommonImplementation::printMeter(Telegram *t,
                                            string *fields, char separator,
                                            string *json,
                                            vector<string> *envs,
-                                           vector<string> *more_json)
+                                           vector<string> *more_json,
+                                           vector<string> *selected_fields)
 {
-    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true);
-    *fields = concatFields(this, t, separator, prints_, conversions_, false);
+    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true, selected_fields);
+    *fields = concatFields(this, t, separator, prints_, conversions_, false, selected_fields);
 
     string s;
     s += "{";

src/meters.h

@@ -88,7 +88,10 @@ struct MeterInfo
 
 struct Meter
 {
+    // This meter listens to these ids.
     virtual vector<string> ids() = 0;
+    // This meter can report these fields, like total_m3, temp_c.
+    virtual vector<string> fields() = 0;
     virtual string meterName() = 0;
     virtual string name() = 0;
     virtual MeterType type() = 0;
@@ -106,7 +109,8 @@ struct Meter
                             string *fields, char separator,
                             string *json,
                             vector<string> *envs,
-                            vector<string> *more_json) = 0;
+                            vector<string> *more_json,
+                            vector<string> *selected_fields) = 0;
 
     // The handleTelegram expects an input_frame where the DLL crcs have been removed.
     bool handleTelegram(vector<uchar> input_frame);

src/meters_common_implementation.h

@@ -39,6 +39,7 @@ struct Print
 struct MeterCommonImplementation : public virtual Meter
 {
     vector<string> ids();
+    vector<string> fields();
     string name();
     MeterType type();
     vector<int> media();
@@ -94,7 +95,8 @@ protected:
                     string *fields, char separator,
                     string *json,
                     vector<string> *envs,
-                    vector<string> *more_json); // Add this json "key"="value" strings.
+                    vector<string> *more_json, // Add this json "key"="value" strings.
+                    vector<string> *selected_fields); // Only print these fields. Json always everything.
 
     virtual void processContent(Telegram *t) = 0;
 
@@ -121,6 +123,7 @@ protected:
     std::map<std::string,std::pair<int,std::string>> values_;
     vector<Unit> conversions_;
     vector<Print> prints_;
+    vector<string> fields_;
 };
 
 #endif

src/printer.cc

@@ -40,13 +40,15 @@ Printer::Printer(bool json, bool fields, char separator,
     timestamp_ = timestamp;
 }
 
-void Printer::print(Telegram *t, Meter *meter, vector<string> *more_json)
+void Printer::print(Telegram *t, Meter *meter,
+                    vector<string> *more_json,
+                    vector<string> *selected_fields)
 {
     string human_readable, fields, json;
     vector<string> envs;
     bool printed = false;
 
-    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json);
+    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json, selected_fields);
 
     if (shell_cmdlines_.size() > 0 || meter->shellCmdlines().size() > 0) {
         printShells(meter, envs);

src/printer.h

@@ -32,7 +32,7 @@ struct Printer {
             MeterFileNaming naming,
             MeterFileTimestamp timestamp);
 
-    void print(Telegram *t, Meter *meter, vector<string> *more_json);
+    void print(Telegram *t, Meter *meter, vector<string> *more_json, vector<string> *selected_fields);
 
     private:
 

test.sh

@@ -48,6 +48,9 @@ if [ "$?" != "0" ]; then RC="1"; fi
 tests/test_conversions.sh $PROG
 if [ "$?" != "0" ]; then RC="1"; fi
 
+tests/test_fields.sh $PROG
+if [ "$?" != "0" ]; then RC="1"; fi
+
 #tests/test_oneshot.sh $PROG broken test
 #if [ "$?" != "0" ]; then RC="1"; fi
 

tests/test_additional_json.sh

@@ -29,7 +29,7 @@ if [ "$TESTRESULT" = "ERROR" ]; then echo ERROR: $TESTNAME;  exit 1; fi
 TESTNAME="Test additional shell envs from cmdline"
 TESTRESULT="ERROR"
 
-$PROG --format=json --json_floor=5 --json_house="alfa beta" --shellenvs --listento=c1 simulations/simulation_additional_json.txt \
+$PROG --format=json --json_floor=5 --json_house="alfa beta" --listenvs --listento=c1 simulations/simulation_additional_json.txt \
       Vatten multical21 76348799 "" | grep METER_  > $TEST/test_output.txt
 
 ENVS=$(cat $TEST/test_output.txt | tr '\n' ' ')

tests/test_fields.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+PROG="$1"
+
+mkdir -p testoutput
+
+TEST=testoutput
+
+TESTNAME="Test selected fields"
+TESTRESULT="ERROR"
+
+cat <<EOF > $TEST/test_expected.txt
+76348799;Vatten;6408;6.408;0;127;260.6
+76348799;Vatten;6408;6.408;0;127;260.6
+EOF
+
+$PROG --format=fields --separator=';' \
+      --selectfields=id,name,total_l,total_m3,max_flow_m3h,flow_temperature_c,flow_temperature_f \
+      --addconversions=L,F \
+      simulations/simulation_c1.txt Vatten multical21 76348799 "" \
+      > $TEST/test_output.txt
+
+if [ "$?" = "0" ]
+then
+    diff $TEST/test_expected.txt $TEST/test_output.txt
+    if [ "$?" = "0" ]
+    then
+        echo OK: $TESTNAME
+        TESTRESULT="OK"
+    fi
+fi
+
+if [ "$TESTRESULT" = "ERROR" ]
+then
+    echo ERROR: $TESTNAME
+    exit 1
+fi

wmbusmeters.1

@@ -57,7 +57,7 @@ mqtt_publish) sent to a REST API (eg curl) or store it in a database
 
 \fB\--shell=\fR<cmdline> invokes cmdline with env variables containing the latest reading
 
-\fB\--shellenvs\fR list the env variables available for the meter
+\fB\--listenvs\fR list the env variables available for the meter
 
 \fB\--useconfig=\fR<dir> load config files from dir/etc