Add recovery when rawtty gets out of sync.

Makefile

@@ -166,11 +166,9 @@ clean:
 	rm -rf build/* build_arm/* build_debug/* build_arm_debug/* *~
 
 test:
-	./build/testinternals
 	./test.sh build/wmbusmeters
 
 testd:
-	./build_debug/testinternals
 	./test.sh build_debug/wmbusmeters
 
 update_manufacturers:

simulations/serial_rawtty_bad.hex

@@ -0,0 +1,2 @@
+1122332e44333003020100071b7a634820252f2f0265840842658308820165950802fb1aae0142fb1aae018201fb1aa9012f
+11223344555744b40988227711101b7ab20800000265a00842658f088201659f08226589081265a0086265510852652b0902fb1aba0142fb1ab0018201fb1abd0122fb1aa90112fb1aba0162fb1aa60152fb1af501066d3b3bb36b2a00

simulations/serial_rawtty_ok.hex

@@ -0,0 +1,2 @@
+2e44333003020100071b7a634820252f2f0265840842658308820165950802fb1aae0142fb1aae018201fb1aa9012f
+5744b40988227711101b7ab20800000265a00842658f088201659f08226589081265a0086265510852652b0902fb1aba0142fb1ab0018201fb1abd0122fb1aa90112fb1aba0162fb1aa60152fb1af501066d3b3bb36b2a00

src/main.cc

@@ -143,6 +143,11 @@ bool startUsingCommandline(Configuration *config)
     }
     verbose("(config) number of meters: %d\n", config->meters.size());
 
+    bool use_stdin = false;
+    if (config->device == "stdin")
+    {
+        use_stdin = true;
+    }
     auto manager = createSerialCommunicationManager(config->exitafter, config->reopenafter);
     onExit(call(manager.get(),stop));
 
@@ -275,6 +280,28 @@ LIST_OF_METERS
     if (type_and_device.first == DEVICE_SIMULATOR) {
         wmbus->simulate();
     }
+    if (use_stdin)
+    {
+        // It will only do a single read from stdin and then stop.
+        // This is for testing (fuzzing).
+        manager->stop();
+        vector<uchar> data;
+        data.resize(1024);
+        int n = read(0, &((data)[0]), 1024);
+        if (n > 0)
+        {
+            data.resize(n);
+            if (isDebugEnabled()) {
+                string msg = bin2hex(data);
+                debug("(serialstdin) received \"%s\"\n", msg.c_str());
+            }
+            wmbus->serial()->fill(data);
+        }
+        else
+        {
+            warning("(serialstdin) nothing received!\n");
+        }
+    }
 
     if (config->daemon) {
         notice("(wmbusmeters) waiting for telegrams\n");

src/serial.cc

@@ -40,7 +40,8 @@ struct SerialDeviceTTY;
 struct SerialDeviceCommand;
 struct SerialDeviceSimulator;
 
-struct SerialCommunicationManagerImp : public SerialCommunicationManager {
+struct SerialCommunicationManagerImp : public SerialCommunicationManager
+{
     SerialCommunicationManagerImp(time_t exit_after_seconds, time_t reopen_after_seconds);
     ~SerialCommunicationManagerImp() { }
 
@@ -77,6 +78,7 @@ struct SerialDeviceImp : public SerialDevice {
 
     int fd() { return fd_; }
     bool working() { return true; }
+    void fill(vector<uchar> &data) {};
 
     protected:
 
@@ -393,22 +395,33 @@ int SerialDeviceCommand::receive(vector<uchar> *data)
 struct SerialDeviceSimulator : public SerialDeviceImp
 {
     SerialDeviceSimulator(SerialCommunicationManagerImp *m) :
-        manager_(m) {};
+        manager_(m) {
+        manager_->opened(this);
+        verbose("(serialsimulator) opened\n");
+    };
     ~SerialDeviceSimulator() {};
 
     bool open(bool fail_if_not_ok) { return true; };
     void close() { };
     void checkIfShouldReopen() { }
     bool send(vector<uchar> &data) { return true; };
-    int receive(vector<uchar> *data) { return 0; };
+    void fill(vector<uchar> &data) { data_ = data; on_data_(); }; // Fill buffer and trigger callback.
+
+    int receive(vector<uchar> *data)
+    {
+        *data = data_;
+        data_.clear();
+        return data->size();
+    }
     int fd() { return 0; }
-    bool working() { return true; }
+    bool working() { return false; } // Only one message that has already been handled! So return false here.
 
     SerialCommunicationManager *manager() { return manager_; }
 
     private:
 
     SerialCommunicationManagerImp *manager_;
+    vector<uchar> data_;
 };
 
 SerialCommunicationManagerImp::SerialCommunicationManagerImp(time_t exit_after_seconds,
@@ -431,6 +444,10 @@ void *SerialCommunicationManagerImp::startLoop(void *a) {
 unique_ptr<SerialDevice> SerialCommunicationManagerImp::createSerialDeviceTTY(string device,
                                                                               int baud_rate)
 {
+    if (device == "stdin")
+    {
+        return unique_ptr<SerialDevice>(new SerialDeviceSimulator(this));
+    }
     return unique_ptr<SerialDevice>(new SerialDeviceTTY(device, baud_rate, this));
 }
 
@@ -525,7 +542,6 @@ void *SerialCommunicationManagerImp::eventLoop() {
         }
 
         int activity = select(max_fd_+1 , &readfds , NULL , NULL, &timeout);
-
         if (!running_) break;
         if (activity < 0 && errno!=EINTR) {
             warning("(serialtty) internal error after select! errno=%s\n", strerror(errno));
@@ -534,7 +550,9 @@ void *SerialCommunicationManagerImp::eventLoop() {
             for (SerialDevice *d : devices_) {
                 if (FD_ISSET(d->fd(), &readfds)) {
                     SerialDeviceImp *si = dynamic_cast<SerialDeviceImp*>(d);
-                    if (si->on_data_) si->on_data_();
+                    if (si->on_data_) {
+                        si->on_data_();
+                    }
                 }
             }
         }

src/serial.h

@@ -38,6 +38,7 @@ struct SerialDevice
     virtual int receive(std::vector<uchar> *data) = 0;
     virtual int fd() = 0;
     virtual bool working() = 0;
+    virtual void fill(std::vector<uchar> &data) = 0; // Fill buffer with raw data.
     virtual SerialCommunicationManager *manager() = 0;
     virtual ~SerialDevice() = default;
 };

src/testinternals.cc

@@ -39,6 +39,7 @@ int main(int argc, char **argv)
         // Supply --debug (oh well, anything works) to enable debug mode.
         debugEnabled(true);
     }
+    onExit([](){});
     test_crc();
     test_dvparser();
     test_linkmodes();
@@ -208,7 +209,7 @@ int test_linkmodes()
     unique_ptr<WMBus> wmbus_im871a = openIM871A("", manager.get(), serial1.release());
     unique_ptr<WMBus> wmbus_amb8465 = openAMB8465("", manager.get(), serial2.release());
     unique_ptr<WMBus> wmbus_rtlwmbus = openRTLWMBUS("", manager.get(), serial3.release(), [](){});
-    unique_ptr<WMBus> wmbus_rawtty = openRawTTY("/dev/ttyACM", manager.get(), serial4.release());
+    unique_ptr<WMBus> wmbus_rawtty = openRawTTY("", manager.get(), serial4.release());
 
     Configuration nometers_config;
     // Check that if no meters are supplied then you must set a link mode.

src/util.cc

@@ -594,6 +594,9 @@ bool checkCharacterDeviceExists(const char *tty, bool fail_if_not)
 {
     struct stat info;
 
+    // Stdin always exists.
+    if (!strcmp(tty, "stdin")) return true;
+
     int rc = stat(tty, &info);
     if (rc != 0) {
         if (fail_if_not) {

src/wmbus.cc

@@ -413,7 +413,9 @@ bool detectAMB8465(string device, SerialCommunicationManager *handler);
 bool detectRawTTY(string device, SerialCommunicationManager *handler);
 bool detectRTLSDR(string device, SerialCommunicationManager *handler);
 
-pair<MBusDeviceType,string> detectMBusDevice(string device, string suffix, SerialCommunicationManager *handler)
+pair<MBusDeviceType,string> detectMBusDevice(string device,
+                                             string suffix,
+                                             SerialCommunicationManager *handler)
 {
     if (device == "rtlwmbus")
     {
@@ -477,8 +479,21 @@ pair<MBusDeviceType,string> detectMBusDevice(string device, string suffix, Seria
         return { DEVICE_SIMULATOR, device };
     }
 
-    // If not auto, then test the device, is it a character device?
-    checkCharacterDeviceExists(device.c_str(), true);
+    if (device == "stdin")
+    {
+        // The stdin device is only for testing (fuzzing).
+        if (suffix == "")
+        {
+            // Assuming rawtty.
+            return { DEVICE_RAWTTY, device };
+        }
+        // Otherwise check the suffix below.
+    }
+    else
+    {
+        // If not auto and not stdin, then test the device, is it a character device?
+        checkCharacterDeviceExists(device.c_str(), true);
+    }
 
     if (suffix != "")
     {

src/wmbus.h

@@ -212,7 +212,8 @@ LIST_OF_MBUS_DEVICES
 
 // The detect function can be supplied the device "auto" and will try default locations for the device.
 // Returned is the type and the found device string.
-pair<MBusDeviceType,string> detectMBusDevice(string devstr, string suffix, SerialCommunicationManager *manager);
+pair<MBusDeviceType,string> detectMBusDevice(string devstr, string suffix,
+                                             SerialCommunicationManager *manager);
 
 unique_ptr<WMBus> openIM871A(string device, SerialCommunicationManager *manager);
 unique_ptr<WMBus> openIM871A(string device, SerialCommunicationManager *manager, SerialDevice *serial);

src/wmbus_rawtty.cc

@@ -128,12 +128,49 @@ FrameStatus WMBusRawTTY::checkRawTTYFrame(vector<uchar> &data,
                                           int *payload_len_out,
                                           int *payload_offset)
 {
-    // telegram=|2A442D2C998734761B168D2021D0871921|58387802FF2071000413F81800004413F8180000615B|+96
+    // Nice clean: 2A442D2C998734761B168D2021D0871921|58387802FF2071000413F81800004413F8180000615B
+    // Ugly: 00615B2A442D2C998734761B168D2021D0871921|58387802FF2071000413F81800004413F8180000615B
+    // Here the frame is prefixed with some random data.
+
     if (data.size() == 0) return PartialFrame;
     int payload_len = data[0];
+    int type = data[1];
+    int offset = 1;
+
+    if (type != 0x44)
+    {
+        // Ouch, we are out of sync with the wmbus frames that we expect!
+        // Since we currently do not handle any other type of frame, we can
+        // look for the byte 0x44 in the buffer. If we find a 0x44 byte and
+        // the length byte before it maps to the end of the buffer,
+        // then we have found a valid telegram.
+        bool found = false;
+        for (size_t i = 0; i < data.size()-2; ++i)
+        {
+            if (data[i+1] == 0x44)
+            {
+                payload_len = data[i];
+                size_t remaining = data.size()-i;
+                if (data[i]+1 == (uchar)remaining && data[i+1] == 0x44)
+                {
+                    found = true;
+                    offset = i+1;
+                    verbose("(wmbus_rawtty) out of sync, skipping %d bytes.\n", (int)i);
+                    break;
+                }
+            }
+        }
+        if (!found)
+        {
+            // No sensible telegram in the buffer. Flush it!
+            verbose("(wmbus_rawtty) no sensible telegram found, clearing buffer.\n");
+            data.clear();
+            return ErrorInFrame;
+        }
+    }
     *payload_len_out = payload_len;
-    *payload_offset = 1;
-    *frame_length = payload_len+1;
+    *payload_offset = offset;
+    *frame_length = payload_len+offset;
     if (data.size() < *frame_length) return PartialFrame;
 
     return FullFrame;
@@ -148,28 +185,42 @@ void WMBusRawTTY::processSerialData()
 
     read_buffer_.insert(read_buffer_.end(), data.begin(), data.end());
 
-    size_t frame_length;
-    int payload_len, payload_offset;
+    for (;;)
+    {
+        size_t frame_length;
+        int payload_len, payload_offset;
 
-    FrameStatus status = checkRawTTYFrame(read_buffer_, &frame_length, &payload_len, &payload_offset);
+        FrameStatus status = checkRawTTYFrame(read_buffer_, &frame_length, &payload_len, &payload_offset);
 
-    if (status == ErrorInFrame) {
-        verbose("(rawtty) protocol error in message received!\n");
-        string msg = bin2hex(read_buffer_);
-        debug("(rawtty) protocol error \"%s\"\n", msg.c_str());
-        read_buffer_.clear();
-    } else
-    if (status == FullFrame) {
-
-        vector<uchar> payload;
-        if (payload_len > 0) {
-            uchar l = payload_len;
-            payload.insert(payload.end(), &l, &l+1); // Re-insert the len byte.
-            payload.insert(payload.end(), read_buffer_.begin()+payload_offset, read_buffer_.begin()+payload_offset+payload_len);
+        if (status == PartialFrame)
+        {
+            // Partial frame, stop eating.
+            break;
+        }
+        else if (status == ErrorInFrame)
+        {
+            verbose("(rawtty) protocol error in message received!\n");
+            string msg = bin2hex(read_buffer_);
+            debug("(rawtty) protocol error \"%s\"\n", msg.c_str());
+            read_buffer_.clear();
+            break;
+        }
+        else if (status == FullFrame)
+        {
+            vector<uchar> payload;
+            if (payload_len > 0)
+            {
+                uchar l = payload_len;
+                payload.insert(payload.end(), &l, &l+1); // Re-insert the len byte.
+                payload.insert(payload.end(), read_buffer_.begin()+payload_offset, read_buffer_.begin()+payload_offset+payload_len);
+            }
+            read_buffer_.erase(read_buffer_.begin(), read_buffer_.begin()+frame_length);
+            handleMessage(payload);
+        }
+        else
+        {
+            assert(0);
         }
-
-        read_buffer_.erase(read_buffer_.begin(), read_buffer_.begin()+frame_length);
-        handleMessage(payload);
     }
 }
 

test.sh

@@ -24,4 +24,5 @@ tests/test_wrongkeys.sh $PROG
 tests/test_config4.sh $PROG
 tests/test_linkmodes.sh $PROG
 tests/test_additional_json.sh $PROG
+tests/test_serial_bads.sh $PROG
 tests/test_rtlwmbus.sh $PROG

tests/test_serial_bads.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+PROG="$1"
+
+mkdir -p testoutput
+
+TEST=testoutput
+
+xxd -r -p simulations/serial_rawtty_ok.hex | \
+    $PROG --format=json --listento=any stdin \
+          Rummet1 lansenth 00010203 "" \
+          Rummet2 rfmamb 11772288 "" \
+    | grep Rummet > $TEST/test_output.txt
+
+
+cat > $TEST/test_expected.txt <<EOF
+{"media":"room sensor","meter":"lansenth","name":"Rummet1","id":"00010203","current_temperature_c":21.8,"current_relative_humidity_rh":43,"average_temperature_1h_c":21.79,"average_relative_humidity_1h_rh":43,"average_temperature_24h_c":21.97,"average_relative_humidity_24h_rh":42.5,"timestamp":"1111-11-11T11:11:11Z"}
+{"media":"room sensor","meter":"rfmamb","name":"Rummet2","id":"11772288","current_temperature_c":22.08,"average_temperature_1h_c":21.91,"average_temperature_24h_c":22.07,"maximum_temperature_1h_c":22.08,"minimum_temperature_1h_c":21.85,"maximum_temperature_24h_c":23.47,"minimum_temperature_24h_c":21.29,"current_relative_humidity_rh":44.2,"average_relative_humidity_1h_rh":43.2,"average_relative_humidity_24h_rh":44.5,"minimum_relative_humidity_1h_rh":42.2,"maximum_relative_humidity_1h_rh":50.1,"maximum_relative_humidity_24h_rh":0,"minimum_relative_humidity_24h_rh":0,"device_date_time":"2019-10-11 19:59","timestamp":"1111-11-11T11:11:11Z"}
+EOF
+if [ "$?" == "0" ]
+then
+    cat $TEST/test_output.txt | sed 's/"timestamp":"....-..-..T..:..:..Z"/"timestamp":"1111-11-11T11:11:11Z"/' > $TEST/test_responses.txt
+    diff $TEST/test_expected.txt $TEST/test_responses.txt
+    if [ "$?" == "0" ]
+    then
+        echo Serial simulation OK
+    fi
+else
+    echo Failure.
+    exit 1
+fi
+
+xxd -r -p simulations/serial_rawtty_bad.hex | \
+    $PROG --format=json --listento=any stdin \
+          Rummet1 lansenth 00010203 "" \
+          Rummet2 rfmamb 11772288 "" \
+    | grep Rummet > $TEST/test_output.txt
+
+
+cat > $TEST/test_expected.txt <<EOF
+{"media":"room sensor","meter":"rfmamb","name":"Rummet2","id":"11772288","current_temperature_c":22.08,"average_temperature_1h_c":21.91,"average_temperature_24h_c":22.07,"maximum_temperature_1h_c":22.08,"minimum_temperature_1h_c":21.85,"maximum_temperature_24h_c":23.47,"minimum_temperature_24h_c":21.29,"current_relative_humidity_rh":44.2,"average_relative_humidity_1h_rh":43.2,"average_relative_humidity_24h_rh":44.5,"minimum_relative_humidity_1h_rh":42.2,"maximum_relative_humidity_1h_rh":50.1,"maximum_relative_humidity_24h_rh":0,"minimum_relative_humidity_24h_rh":0,"device_date_time":"2019-10-11 19:59","timestamp":"1111-11-11T11:11:11Z"}
+EOF
+if [ "$?" == "0" ]
+then
+    cat $TEST/test_output.txt | sed 's/"timestamp":"....-..-..T..:..:..Z"/"timestamp":"1111-11-11T11:11:11Z"/' > $TEST/test_responses.txt
+    diff $TEST/test_expected.txt $TEST/test_responses.txt
+    if [ "$?" == "0" ]
+    then
+        echo Serial simulation with bad prefixes OK
+    fi
+else
+    echo Failure.
+    exit 1
+fi