Added --ignoreduplicates

pull/175/head
Fredrik Öhrström 2020-10-25 20:57:25 +01:00
rodzic aa368b8c49
commit ea530a7531
16 zmienionych plików z 536 dodań i 0 usunięć

Wyświetl plik

@ -1,4 +1,10 @@
Add --ignoreduplicates (ignoreduplicates=true) to ignore
any duplicates. It remembers a history of 10 telegrams.
This is useful when you have multiple dongles for covering
a larger area of reception, but sometimes telegrams arrive
at multiple dongles at the same time.
Add proper support for FlowIQ2200 water meter.
Decode two vendor values in multical603 as energy forward and returned.
Accept t1 and c1 as linkmodes for multical21 meters.

Wyświetl plik

@ -150,6 +150,7 @@ METER_OBJS:=\
$(BUILD)/rtlsdr.o \
$(BUILD)/serial.o \
$(BUILD)/shell.o \
$(BUILD)/sha256.o \
$(BUILD)/threads.o \
$(BUILD)/util.o \
$(BUILD)/units.o \

Wyświetl plik

@ -65,6 +65,7 @@ shell=/usr/bin/mosquitto_pub -h localhost -t wmbusmeters/$METER_ID -m "$METER_JS
alarmshell=/usr/bin/mosquitto_pub -h localhost -t wmbusmeters_alarm -m "$ALARM_TYPE $ALARM_MESSAGE"
alarmtimeout=1h
alarmexpectedactivity=mon-sun(00-23)
ignoreduplicates=false
```
Then add a meter file in /etc/wmbusmeters.d/MyTapWater
@ -155,6 +156,7 @@ As <options> you can use:
--listmeters=<search> list all meter types containing the text <search>
--logfile=<file> use this file instead of stdout
--logtelegrams log the contents of the telegrams for easy replay
--ignoreduplicates ignore duplicate telegrams, remember the last 10 telegrams
--meterfiles=<dir> store meter readings in dir
--meterfilesaction=(overwrite|append) overwrite or append to the meter readings file
--meterfilesnaming=(name|id|name-id) the meter file is the meter's: name, id or name-id

Wyświetl plik

@ -356,6 +356,11 @@ shared_ptr<Configuration> parseCommandLine(int argc, char **argv) {
i++;
continue;
}
if (!strncmp(argv[i], "--ignoreduplicates", 18)) {
c->ignore_duplicate_telegrams = true;
i++;
continue;
}
if (!strncmp(argv[i], "--usestdoutforlogging", 13)) {
c->use_stderr_for_log = false;
i++;

Wyświetl plik

@ -189,6 +189,21 @@ void handleInternalTesting(Configuration *c, string value)
}
}
void handleIgnoreDuplicateTelegrams(Configuration *c, string value)
{
if (value == "true")
{
c->ignore_duplicate_telegrams = true;
}
else if (value == "false")
{
c->ignore_duplicate_telegrams = false;
}
else {
warning("ignoreduplicates should be either true or false, not \"%s\"\n", value.c_str());
}
}
void handleResetAfter(Configuration *c, string s)
{
if (s.length() >= 1)
@ -507,6 +522,7 @@ shared_ptr<Configuration> loadConfiguration(string root, string device_override,
if (p.first.length() > 0 && p.first[0] == '#') continue;
if (p.first == "loglevel") handleLoglevel(c, p.second);
else if (p.first == "internaltesting") handleInternalTesting(c, p.second);
else if (p.first == "ignoreduplicates") handleIgnoreDuplicateTelegrams(c, p.second);
else if (p.first == "device") handleDevice(c, p.second);
else if (p.first == "listento") handleListenTo(c, p.second);
else if (p.first == "logtelegrams") handleLogtelegrams(c, p.second);

Wyświetl plik

@ -66,6 +66,7 @@ struct Configuration
MeterFileTimestamp meterfiles_timestamp {}; // Default is never.
bool use_logfile {};
bool use_stderr_for_log = true; // Default is to use stderr for logging.
bool ignore_duplicate_telegrams = false; // Default is to report all telegrams.
std::string logfile;
bool json {};
bool fields {};

Wyświetl plik

@ -1067,6 +1067,7 @@ bool start(Configuration *config)
traceEnabled(config->trace);
stderrEnabled(config->use_stderr_for_log);
setAlarmShells(config->alarm_shells);
setIgnoreDuplicateTelegrams(config->ignore_duplicate_telegrams);
log_start_information(config);

294
src/sha256.cc 100644
Wyświetl plik

@ -0,0 +1,294 @@
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// WjCryptLib_Sha256
//
// Implementation of SHA256 hash function.
// Original author: Tom St Denis, tomstdenis@gmail.com, http://libtom.org
// Modified by WaterJuice retaining Public Domain license.
//
// This is free and unencumbered software released into the public domain - June 2013 waterjuice.org
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// IMPORTS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#include "sha256.h"
#include <memory.h>
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// MACROS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#define ror(value, bits) (((value) >> (bits)) | ((value) << (32 - (bits))))
#define MIN(x, y) ( ((x)<(y))?(x):(y) )
#define STORE32H(x, y) \
{ (y)[0] = (uint8_t)(((x)>>24)&255); (y)[1] = (uint8_t)(((x)>>16)&255); \
(y)[2] = (uint8_t)(((x)>>8)&255); (y)[3] = (uint8_t)((x)&255); }
#define LOAD32H(x, y) \
{ x = ((uint32_t)((y)[0] & 255)<<24) | \
((uint32_t)((y)[1] & 255)<<16) | \
((uint32_t)((y)[2] & 255)<<8) | \
((uint32_t)((y)[3] & 255)); }
#define STORE64H(x, y) \
{ (y)[0] = (uint8_t)(((x)>>56)&255); (y)[1] = (uint8_t)(((x)>>48)&255); \
(y)[2] = (uint8_t)(((x)>>40)&255); (y)[3] = (uint8_t)(((x)>>32)&255); \
(y)[4] = (uint8_t)(((x)>>24)&255); (y)[5] = (uint8_t)(((x)>>16)&255); \
(y)[6] = (uint8_t)(((x)>>8)&255); (y)[7] = (uint8_t)((x)&255); }
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// CONSTANTS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// The K array
static const uint32_t K[64] = {
0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL,
0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,
0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL,
0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL,
0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL,
0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,
0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL,
0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL,
0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL,
0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,
0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL
};
#define BLOCK_SIZE 64
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// INTERNAL FUNCTIONS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Various logical functions
#define Ch( x, y, z ) (z ^ (x & (y ^ z)))
#define Maj( x, y, z ) (((x | y) & z) | (x & y))
#define S( x, n ) ror((x),(n))
#define R( x, n ) (((x)&0xFFFFFFFFUL)>>(n))
#define Sigma0( x ) (S(x, 2) ^ S(x, 13) ^ S(x, 22))
#define Sigma1( x ) (S(x, 6) ^ S(x, 11) ^ S(x, 25))
#define Gamma0( x ) (S(x, 7) ^ S(x, 18) ^ R(x, 3))
#define Gamma1( x ) (S(x, 17) ^ S(x, 19) ^ R(x, 10))
#define Sha256Round( a, b, c, d, e, f, g, h, i ) \
t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \
t1 = Sigma0(a) + Maj(a, b, c); \
d += t0; \
h = t0 + t1;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// TransformFunction
//
// Compress 512-bits
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
static
void
TransformFunction
(
Sha256Context* Context,
uint8_t const* Buffer
)
{
uint32_t S[8];
uint32_t W[64];
uint32_t t0;
uint32_t t1;
uint32_t t;
int i;
// Copy state into S
for( i=0; i<8; i++ )
{
S[i] = Context->state[i];
}
// Copy the state into 512-bits into W[0..15]
for( i=0; i<16; i++ )
{
LOAD32H( W[i], Buffer + (4*i) );
}
// Fill W[16..63]
for( i=16; i<64; i++ )
{
W[i] = Gamma1( W[i-2]) + W[i-7] + Gamma0( W[i-15] ) + W[i-16];
}
// Compress
for( i=0; i<64; i++ )
{
Sha256Round( S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i );
t = S[7];
S[7] = S[6];
S[6] = S[5];
S[5] = S[4];
S[4] = S[3];
S[3] = S[2];
S[2] = S[1];
S[1] = S[0];
S[0] = t;
}
// Feedback
for( i=0; i<8; i++ )
{
Context->state[i] = Context->state[i] + S[i];
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC FUNCTIONS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Initialise
//
// Initialises a SHA256 Context. Use this to initialise/reset a context.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Initialise
(
Sha256Context* Context // [out]
)
{
Context->curlen = 0;
Context->length = 0;
Context->state[0] = 0x6A09E667UL;
Context->state[1] = 0xBB67AE85UL;
Context->state[2] = 0x3C6EF372UL;
Context->state[3] = 0xA54FF53AUL;
Context->state[4] = 0x510E527FUL;
Context->state[5] = 0x9B05688CUL;
Context->state[6] = 0x1F83D9ABUL;
Context->state[7] = 0x5BE0CD19UL;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Update
//
// Adds data to the SHA256 context. This will process the data and update the internal state of the context. Keep on
// calling this function until all the data has been added. Then call Sha256Finalise to calculate the hash.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Update
(
Sha256Context* Context, // [in out]
void const* Buffer, // [in]
uint32_t BufferSize // [in]
)
{
uint32_t n;
if( Context->curlen > sizeof(Context->buf) )
{
return;
}
while( BufferSize > 0 )
{
if( Context->curlen == 0 && BufferSize >= BLOCK_SIZE )
{
TransformFunction( Context, (uint8_t*)Buffer );
Context->length += BLOCK_SIZE * 8;
Buffer = (uint8_t*)Buffer + BLOCK_SIZE;
BufferSize -= BLOCK_SIZE;
}
else
{
n = MIN( BufferSize, (BLOCK_SIZE - Context->curlen) );
memcpy( Context->buf + Context->curlen, Buffer, (size_t)n );
Context->curlen += n;
Buffer = (uint8_t*)Buffer + n;
BufferSize -= n;
if( Context->curlen == BLOCK_SIZE )
{
TransformFunction( Context, Context->buf );
Context->length += 8*BLOCK_SIZE;
Context->curlen = 0;
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Finalise
//
// Performs the final calculation of the hash and returns the digest (32 byte buffer containing 256bit hash). After
// calling this, Sha256Initialised must be used to reuse the context.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Finalise
(
Sha256Context* Context, // [in out]
SHA256_HASH* Digest // [out]
)
{
int i;
if( Context->curlen >= sizeof(Context->buf) )
{
return;
}
// Increase the length of the message
Context->length += Context->curlen * 8;
// Append the '1' bit
Context->buf[Context->curlen++] = (uint8_t)0x80;
// if the length is currently above 56 bytes we append zeros
// then compress. Then we can fall back to padding zeros and length
// encoding like normal.
if( Context->curlen > 56 )
{
while( Context->curlen < 64 )
{
Context->buf[Context->curlen++] = (uint8_t)0;
}
TransformFunction(Context, Context->buf);
Context->curlen = 0;
}
// Pad up to 56 bytes of zeroes
while( Context->curlen < 56 )
{
Context->buf[Context->curlen++] = (uint8_t)0;
}
// Store length
STORE64H( Context->length, Context->buf+56 );
TransformFunction( Context, Context->buf );
// Copy output
for( i=0; i<8; i++ )
{
STORE32H( Context->state[i], Digest->bytes+(4*i) );
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Calculate
//
// Combines Sha256Initialise, Sha256Update, and Sha256Finalise into one function. Calculates the SHA256 hash of the
// buffer.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Calculate
(
void const* Buffer, // [in]
uint32_t BufferSize, // [in]
SHA256_HASH* Digest // [in]
)
{
Sha256Context context;
Sha256Initialise( &context );
Sha256Update( &context, Buffer, BufferSize );
Sha256Finalise( &context, Digest );
}

91
src/sha256.h 100644
Wyświetl plik

@ -0,0 +1,91 @@
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// WjCryptLib_Sha256
//
// Implementation of SHA256 hash function.
// Original author: Tom St Denis, tomstdenis@gmail.com, http://libtom.org
// Modified by WaterJuice retaining Public Domain license.
//
// This is free and unencumbered software released into the public domain - June 2013 waterjuice.org
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// IMPORTS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
#include <stdint.h>
#include <stdio.h>
#include <memory.h>
typedef struct
{
uint64_t length;
uint32_t state[8];
uint32_t curlen;
uint8_t buf[64];
} Sha256Context;
#define SHA256_HASH_SIZE ( 256 / 8 )
struct SHA256_HASH
{
uint8_t bytes [SHA256_HASH_SIZE];
bool operator == (const SHA256_HASH&o) { return !memcmp(bytes, o.bytes, SHA256_HASH_SIZE); }
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC FUNCTIONS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Initialise
//
// Initialises a SHA256 Context. Use this to initialise/reset a context.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Initialise
(
Sha256Context* Context // [out]
);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Update
//
// Adds data to the SHA256 context. This will process the data and update the internal state of the context. Keep on
// calling this function until all the data has been added. Then call Sha256Finalise to calculate the hash.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Update
(
Sha256Context* Context, // [in out]
void const* Buffer, // [in]
uint32_t BufferSize // [in]
);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Finalise
//
// Performs the final calculation of the hash and returns the digest (32 byte buffer containing 256bit hash). After
// calling this, Sha256Initialised must be used to reuse the context.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Finalise
(
Sha256Context* Context, // [in out]
SHA256_HASH* Digest // [out]
);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Sha256Calculate
//
// Combines Sha256Initialise, Sha256Update, and Sha256Finalise into one function. Calculates the SHA256 hash of the
// buffer.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void
Sha256Calculate
(
void const* Buffer, // [in]
uint32_t BufferSize, // [in]
SHA256_HASH* Digest // [in]
);

Wyświetl plik

@ -16,6 +16,7 @@
*/
#include"aescmac.h"
#include"sha256.h"
#include"timings.h"
#include"meters.h"
#include"wmbus.h"
@ -30,6 +31,9 @@
#include<sys/types.h>
#include<unistd.h>
#include<deque>
#include<algorithm>
struct LinkModeInfo
{
LinkMode mode;
@ -329,6 +333,31 @@ void Telegram::printTPL()
verbose("\n");
}
// Store the hashes of the last 10 telegrams here.
deque<SHA256_HASH> seen_telegrams;
bool seen_this_telegram_before(vector<uchar> &frame)
{
SHA256_HASH hash;
Sha256Calculate(&frame[0], frame.size(), &hash);
auto i = std::find(seen_telegrams.begin(), seen_telegrams.end(), hash);
if (i != seen_telegrams.end())
{
// Found it!
return true;
}
if (seen_telegrams.size() >= 10)
{
seen_telegrams.pop_front();
}
seen_telegrams.push_back(hash);
return false;
}
string manufacturer(int m_field) {
for (auto &m : manufacturers_) {
if (m.m_field == m_field) return m.name;
@ -3202,11 +3231,25 @@ void WMBusCommonImplementation::onTelegram(function<bool(AboutTelegram&,vector<u
telegram_listeners_.push_back(cb);
}
static bool ignore_duplicate_telegrams_ = false;
void setIgnoreDuplicateTelegrams(bool idt)
{
ignore_duplicate_telegrams_ = idt;
}
bool WMBusCommonImplementation::handleTelegram(AboutTelegram &about, vector<uchar> frame)
{
bool handled = false;
last_received_ = time(NULL);
if (ignore_duplicate_telegrams_ && seen_this_telegram_before(frame))
{
verbose("(wmbus) skipping already handled telegram.\n");
return true;
}
for (auto f : telegram_listeners_)
{
if (f)

Wyświetl plik

@ -74,6 +74,8 @@ const char *toString(WMBusDeviceType t);
const char *toLowerCaseString(WMBusDeviceType t);
WMBusDeviceType toWMBusDeviceType(string &t);
void setIgnoreDuplicateTelegrams(bool idt);
struct Detected
{
SpecifiedDevice specified_device {}; // Device as specified from the command line / config file.

Wyświetl plik

@ -84,6 +84,9 @@ if [ "$?" != "0" ]; then RC="1"; fi
tests/test_serial_bads.sh $PROG
if [ "$?" != "0" ]; then RC="1"; fi
tests/test_ignore_duplicates.sh $PROG
if [ "$?" != "0" ]; then RC="1"; fi
if [ "$(uname)" = "Linux" ]
then
tests/test_alarm.sh $PROG

Wyświetl plik

@ -0,0 +1,4 @@
name=Heat
type=multical603
id=36363636
key=

Wyświetl plik

@ -0,0 +1,4 @@
id=52525252
type=flowiq2200
name=MyWater
key=

Wyświetl plik

@ -0,0 +1,61 @@
#!/bin/sh
PROG="$1"
mkdir -p testoutput
TEST=testoutput
####################################################
TESTNAME="Test duplicates are ignored"
TESTRESULT="ERROR"
cat > $TEST/test_expected.txt <<EOF
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
EOF
$PROG --format=json --ignoreduplicates simulations/simulation_duplicates.txt \
Rummet lansensm 01000273 "" \
| grep Rummet > $TEST/test_output.txt
if [ "$?" = "0" ]
then
cat $TEST/test_output.txt | sed 's/"timestamp":"....-..-..T..:..:..Z"/"timestamp":"1111-11-11T11:11:11Z"/' > $TEST/test_responses.txt
diff $TEST/test_expected.txt $TEST/test_responses.txt
if [ "$?" = "0" ]
then
echo "OK: $TESTNAME"
TESTRESULT="OK"
fi
fi
if [ "$TESTRESULT" = "ERROR" ]; then echo ERROR: $TESTNAME; exit 1; fi
####################################################
TESTNAME="Test duplicates are left alone"
TESTRESULT="ERROR"
cat > $TEST/test_expected.txt <<EOF
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
{"media":"smoke detector","meter":"lansensm","name":"Rummet","id":"01000273","status":"OK","timestamp":"1111-11-11T11:11:11Z"}
EOF
$PROG --format=json simulations/simulation_duplicates.txt \
Rummet lansensm 01000273 "" \
| grep Rummet > $TEST/test_output.txt
if [ "$?" = "0" ]
then
cat $TEST/test_output.txt | sed 's/"timestamp":"....-..-..T..:..:..Z"/"timestamp":"1111-11-11T11:11:11Z"/' > $TEST/test_responses.txt
diff $TEST/test_expected.txt $TEST/test_responses.txt
if [ "$?" = "0" ]
then
echo "OK: $TESTNAME"
TESTRESULT="OK"
fi
fi
if [ "$TESTRESULT" = "ERROR" ]; then echo ERROR: $TESTNAME; exit 1; fi

Wyświetl plik

@ -55,6 +55,8 @@ mqtt_publish) sent to a REST API (eg curl) or store it in a database
\fB\--logtelegrams\fR log the contents of the telegrams for easy replay
\fB\--ignoreduplicates\fR ignore duplicate telegrams, remember the last 10 telegrams
\fB\--meterfiles=\fR<dir> store meter readings in dir
\fB\--meterfilesaction=\fR(overwrite|append) overwrite or append to the meter readings file