From 0f209edc9731d21cea08e6d663362fcbc754d229 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fredrik=20=C3=96hrstr=C3=B6m?= <oehrstroem@gmail.com>
Date: Sun, 8 Aug 2021 17:16:52 +0200
Subject: [PATCH] Update README, CHANGES and man page.

---
 CHANGES                            | 19 +++++++++++++++++--
 README.md                          | 22 +++++-----------------
 src/cmdline.cc                     | 12 ------------
 src/config.cc                      | 21 +--------------------
 src/config.h                       |  3 +--
 src/main.cc                        |  5 ++---
 src/meters.cc                      | 14 +++++++-------
 src/meters.h                       |  3 +--
 src/meters_common_implementation.h |  7 +++++--
 src/printer.cc                     |  5 ++---
 src/printer.h                      |  2 +-
 wmbusmeters.1                      |  4 +---
 12 files changed, 43 insertions(+), 74 deletions(-)

diff --git a/CHANGES b/CHANGES
index 102f086..37296d2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,21 @@
 
-Added the field timestamp_ut to print the existing timestamp field
-using unix timestamp seconds since 1970-01-01.
+You can now add constant fields using --field_extra_info=floor54 and
+--selectfields=....,extra_info,.... to print constant values into
+the fields format. --field_xxx=yyy is the new name for --json_xxx=yyy.
+The old --json_xxx=yyy continues to work for backwards compatibility.
+
+The timestamp field/key is the time registered by wmbusmeters
+when the telegram was received. (The telegram itself sometimes
+contain another timestamp when the telegram was sent.)
+
+This timestamp field is encoded using the local time when used
+in hr/fields (ie "2021-01-01 12:15.00") but UTC time within json
+(ie "2021-01-01T10:15.00Z").
+
+This will stay the same. However now you can explicitly select
+the timestamp format for the selected fields. Select timestamp_ut
+to print the timestamp as a unix timestamp (ie seconds since 1970-01-01).
+Select timestamp_utc for the timestamp in UTC and timestamp_lt for local time.
 
 Daniel Glaser added support for the Aventies water meter! Thanks Daniel!
 
diff --git a/README.md b/README.md
index e7780af..1a49d82 100644
--- a/README.md
+++ b/README.md
@@ -214,9 +214,7 @@ As <options> you can use:
     --nodeviceexit if no wmbus devices are found, then exit immediately
     --oneshot wait for an update from each meter, then quit
     --resetafter=<time> reset the wmbus dongle regularly, default is 23h
-    --addfields=timestamp_ut add the timestamp_ut field to be printed (--listfields=<meter> to list available fields)
     --selectfields=id,timestamp,total_m3 select only these fields to be printed (--listfields=<meter> to list available fields)
-      addfields and selectfields are mutually exclusive.
     --separator=<c> change field separator to c
     --shell=<cmdline> invokes cmdline with env variables containing the latest reading
     --silent do not print informational messages nor warnings
@@ -453,28 +451,18 @@ wmbusmeters --format=fields 'rtlwmbus(ppm=72)' GreenhouseWater multical21:c1 333
 GreenhouseTapWater;33333333;9999.099;77.712;0.000;11;31;;2018-03-05 12:10.24
 ```
 
-You can select a subset of all available fields:
+You can select a subset of all available fields, where we also select to print the timestamp as a unix timestamp.
+The timestamp field is UTC time for json and local time when hr and fields. To explicitly select utc you
+can specify timestamp_utc and timestamp_lt for local time.
 
 ```shell
-wmbusmeters --format=fields --selectfields=id,total_m3 /dev/ttyUSB0:im871a GreenhouseWater multical21:c1 33333333 NOKEY
+wmbusmeters --format=fields --selectfields=id,total_m3,timestamp_ut,timestamp_utc /dev/ttyUSB0:im871a GreenhouseWater multical21:c1 33333333 NOKEY
 ```
 
 ```
-33333333;9999.099
+33333333;9999.099;1628434800;2021-08-08T15:00.00Z
 ```
 
-Instead of selecting a subset you can add fields, such as: timestamp_ut, timestamp_utc, timestamp_lt
-and extra constant fields added using --field_xxx=yyy.
-
-```shell
-wmbusmeters --format=fields --field_city=Stockholm --addfields=timestamp_ut,city /dev/ttyUSB0:im871a GreenhouseWater multical21:c1 33333333 NOKEY
-```
-
-```
-GreenhouseTapWater;33333333;9999.099;77.712;0.000;11;31;;2018-03-05 12:10.24;1627855621,Stockholm
-```
-
-
 You can list all available fields for a meter: `wmbusmeters --listfields=multical21`
 
 You can list all meters: `wmbusmeters --listmeters`
diff --git a/src/cmdline.cc b/src/cmdline.cc
index f175a65..286ff38 100644
--- a/src/cmdline.cc
+++ b/src/cmdline.cc
@@ -277,18 +277,6 @@ shared_ptr<Configuration> parseCommandLine(int argc, char **argv) {
             i++;
             continue;
         }
-        if (!strncmp(argv[i], "--addfields=", 12)) {
-            if (strlen(argv[i]) > 12)
-            {
-                string s = string(argv[i]+12);
-                handleAddedFields(c, s);
-            } else {
-                error("You must supply fields to be added.\n");
-            }
-            i++;
-            continue;
-        }
-
         if (!strncmp(argv[i], "--separator=", 12)) {
             if (!c->fields) {
                 error("You must specify --format=fields before --separator=X\n");
diff --git a/src/config.cc b/src/config.cc
index ea983a8..be8d4bc 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -553,24 +553,6 @@ void handleLogTimestamps(Configuration *c, string ts)
 }
 
 void handleSelectedFields(Configuration *c, string s)
-{
-    if (c->added_fields.size() > 0)
-    {
-        warning("(warning) addfields already used! Ignoring addfields %s", s);
-        return;
-    }
-    char buf[s.length()+1];
-    strcpy(buf, s.c_str());
-    char *saveptr {};
-    const char *tok = strtok_r(buf, ",", &saveptr);
-    while (tok != NULL)
-    {
-        c->selected_fields.push_back(tok);
-        tok = strtok_r(NULL, ",", &saveptr);
-    }
-}
-
-void handleAddedFields(Configuration *c, string s)
 {
     if (c->selected_fields.size() > 0)
     {
@@ -583,7 +565,7 @@ void handleAddedFields(Configuration *c, string s)
     const char *tok = strtok_r(buf, ",", &saveptr);
     while (tok != NULL)
     {
-        c->added_fields.push_back(tok);
+        c->selected_fields.push_back(tok);
         tok = strtok_r(NULL, ",", &saveptr);
     }
 }
@@ -646,7 +628,6 @@ shared_ptr<Configuration> loadConfiguration(string root, string device_override,
         else if (p.first == "addconversions") handleConversions(c, p.second);
         else if (p.first == "logtimestamps") handleLogTimestamps(c, p.second);
         else if (p.first == "selectfields") handleSelectedFields(c, p.second);
-        else if (p.first == "addfields") handleAddedFields(c, p.second);
         else if (p.first == "shell") handleShell(c, p.second);
         else if (p.first == "resetafter") handleResetAfter(c, p.second);
         else if (p.first == "alarmshell") handleAlarmShell(c, p.second);
diff --git a/src/config.h b/src/config.h
index 024b2ef..22d1de8 100644
--- a/src/config.h
+++ b/src/config.h
@@ -110,10 +110,9 @@ struct Configuration
     bool no_init {};
     std::vector<Unit> conversions;
     std::vector<std::string> selected_fields;
-    std::vector<std::string> added_fields;
     std::vector<MeterInfo> meters;
     std::vector<std::string> extra_constant_fields; // Additional constant fields to always add to json.
-    // These extra constant fields can also be part of selected with selectfields or added with addfields.
+    // These extra constant fields can also be part of selected with selectfields.
 
     ~Configuration() = default;
 };
diff --git a/src/main.cc b/src/main.cc
index 84a4351..af8e9a4 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -188,8 +188,7 @@ void list_shell_envs(Configuration *config, string meter_driver)
                       &ignore3,
                       &envs,
                       &config->extra_constant_fields,
-                      &config->selected_fields,
-                      &config->added_fields);
+                      &config->selected_fields);
 
     for (auto &e : envs)
     {
@@ -408,7 +407,7 @@ bool start(Configuration *config)
     meter_manager_->whenMeterUpdated(
         [&](Telegram *t,Meter *meter)
         {
-            printer_->print(t, meter, &config->extra_constant_fields, &config->selected_fields, &config->added_fields);
+            printer_->print(t, meter, &config->extra_constant_fields, &config->selected_fields);
             oneshot_check(config, t, meter);
         }
     );
diff --git a/src/meters.cc b/src/meters.cc
index 44a371d..64e9e1d 100644
--- a/src/meters.cc
+++ b/src/meters.cc
@@ -585,7 +585,7 @@ void MeterCommonImplementation::triggerUpdate(Telegram *t)
 }
 
 string concatAllFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr,
-                       vector<string> *added_fields, vector<string> *extra_constant_fields)
+                       vector<string> *extra_constant_fields)
 {
     string s;
     s = "";
@@ -747,11 +747,11 @@ bool checkConstantField(string *buf, string field, char c, vector<string> *extra
 
 
 string concatFields(Meter *m, Telegram *t, char c, vector<Print> &prints, vector<Unit> &cs, bool hr,
-                    vector<string> *selected_fields, vector<string> *added_fields, vector<string> *extra_constant_fields)
+                    vector<string> *selected_fields, vector<string> *extra_constant_fields)
 {
     if (selected_fields == NULL || selected_fields->size() == 0)
     {
-        return concatAllFields(m, t, c, prints, cs, hr, added_fields, extra_constant_fields);
+        return concatAllFields(m, t, c, prints, cs, hr, extra_constant_fields);
     }
     string buf = "";
 
@@ -831,11 +831,10 @@ void MeterCommonImplementation::printMeter(Telegram *t,
                                            string *json,
                                            vector<string> *envs,
                                            vector<string> *extra_constant_fields,
-                                           vector<string> *selected_fields,
-                                           vector<string> *added_fields)
+                                           vector<string> *selected_fields)
 {
-    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true, selected_fields, added_fields, extra_constant_fields);
-    *fields = concatFields(this, t, separator, prints_, conversions_, false, selected_fields, added_fields, extra_constant_fields);
+    *human_readable = concatFields(this, t, '\t', prints_, conversions_, true, selected_fields, extra_constant_fields);
+    *fields = concatFields(this, t, separator, prints_, conversions_, false, selected_fields, extra_constant_fields);
 
     string media;
     if (t->tpl_id_found)
@@ -886,6 +885,7 @@ void MeterCommonImplementation::printMeter(Telegram *t,
         }
     }
     s += "\"timestamp\":\""+datetimeOfUpdateRobot()+"\"";
+
     if (t->about.device != "")
     {
         s += ",";
diff --git a/src/meters.h b/src/meters.h
index b494bb2..6be5343 100644
--- a/src/meters.h
+++ b/src/meters.h
@@ -230,8 +230,7 @@ struct Meter
                             string *json,
                             vector<string> *envs,
                             vector<string> *more_json,
-                            vector<string> *selected_fields,
-                            vector<string> *added_fields) = 0;
+                            vector<string> *selected_fields) = 0;
 
     // The handleTelegram expects an input_frame where the DLL crcs have been removed.
     // Returns true of this meter handled this telegram!
diff --git a/src/meters_common_implementation.h b/src/meters_common_implementation.h
index 0fe480a..b021926 100644
--- a/src/meters_common_implementation.h
+++ b/src/meters_common_implementation.h
@@ -89,8 +89,11 @@ protected:
                     string *json,
                     vector<string> *envs,
                     vector<string> *more_json, // Add this json "key"="value" strings.
-                    vector<string> *selected_fields, // Only print these fields. Json always everything.
-                    vector<string> *added_fields); // Added these fields for printing.
+                    vector<string> *selected_fields); // Only print these fields.
+    // Json fields cannot be modified expect by adding conversions.
+    // Json fields include all values except timestamp_ut, timestamp_utc, timestamp_lt
+    // since Json is assumed to be decoded by a program and the current timestamp which is the
+    // same as timestamp_utc, can always be decoded/recoded into local time or a unix timestamp.
 
     virtual void processContent(Telegram *t) = 0;
 
diff --git a/src/printer.cc b/src/printer.cc
index 0988f08..ffde9d1 100644
--- a/src/printer.cc
+++ b/src/printer.cc
@@ -42,14 +42,13 @@ Printer::Printer(bool json, bool fields, char separator,
 
 void Printer::print(Telegram *t, Meter *meter,
                     vector<string> *more_json,
-                    vector<string> *selected_fields,
-                    vector<string> *added_fields)
+                    vector<string> *selected_fields)
 {
     string human_readable, fields, json;
     vector<string> envs;
     bool printed = false;
 
-    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json, selected_fields, added_fields);
+    meter->printMeter(t, &human_readable, &fields, separator_, &json, &envs, more_json, selected_fields);
 
     if (shell_cmdlines_.size() > 0 || meter->shellCmdlines().size() > 0) {
         printShells(meter, envs);
diff --git a/src/printer.h b/src/printer.h
index 4c69a75..fa1ec99 100644
--- a/src/printer.h
+++ b/src/printer.h
@@ -32,7 +32,7 @@ struct Printer {
             MeterFileNaming naming,
             MeterFileTimestamp timestamp);
 
-    void print(Telegram *t, Meter *meter, vector<string> *more_json, vector<string> *selected_fields, vector<string> *added_fields);
+    void print(Telegram *t, Meter *meter, vector<string> *more_json, vector<string> *selected_fields);
 
     private:
 
diff --git a/wmbusmeters.1 b/wmbusmeters.1
index 382a376..95f98a2 100644
--- a/wmbusmeters.1
+++ b/wmbusmeters.1
@@ -43,7 +43,7 @@ mqtt_publish) sent to a REST API (eg curl) or store it in a database
 
 \fB\--ignoreduplicates\fR=<bool> ignore duplicate telegrams, remember the last 10 telegrams. Default is true.
 
-\fB\--field_xxx=yyy\fR always add "xxx"="yyy" to the json output and add shell env METER_xxx=yyy The field xxx can also be selected or added using selectfields= and addfields=. Equivalent command is --json_xxx=yyy.
+\fB\--field_xxx=yyy\fR always add "xxx"="yyy" to the json output and add shell env METER_xxx=yyy The field xxx can also be selected or added using selectfields=. Equivalent older command is --json_xxx=yyy.
 
 \fB\--license\fR print GPLv3+ license
 
@@ -79,8 +79,6 @@ mqtt_publish) sent to a REST API (eg curl) or store it in a database
 
 \fB\--resetafter=\fR<time> reset the wmbus dongle regularly, default is 23h
 
-\fB\--addfields=\fRtimestamp_ut add the timestamp_ut field to be printed (--listfields=<meter> to list available fields)
-
 \fB\--selectfields=\fRid,timestamp,total_m3 select only these fields to be printed (--listfields=<meter> to list available fields)
 
 \fB\--separator=\fR<c> change field separator to c