Cleanup and refactor.

koppelting
Bertrik Sikken 2017-08-22 10:54:38 +02:00
rodzic 37f63f47d1
commit 421448f645
20 zmienionych plików z 198 dodań i 118 usunięć

Wyświetl plik

@ -0,0 +1,17 @@
# Root logger option
log4j.rootLogger=INFO, stdout
# Direct log messages to stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
#log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1}:%L - %m%n
# Direct log messages to file
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=ttnhabbridge.log
log4j.appender.file.MaxFileSize=10MB
log4j.appender.file.MaxBackupIndex=10
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{1}:%L - %m%n

Wyświetl plik

@ -0,0 +1,21 @@
# URL of the habitat server
habitat.url=http://habitat.habhub.org
# Timeout in milliseconds
habitat.timeout=3000
# URL of the TTN MQTT server
mqtt.serverurl=tcp://eu.thethings.network
# MQTT client id
mqtt.clientid=ttnhabbridge
# TTN application name used as MQTT user name
mqtt.username=ttnmapper
# TTN application password
mqtt.password=ttn-account-v2.Xc8BFRKeBK5nUhc9ikDcR-sbelgSMdHKnOQKMAiwpgI
# MQTT topic to subscribe to
mqtt.topic=ttnmapper/devices/+/up

Wyświetl plik

@ -25,11 +25,16 @@ import nl.sikken.bertrik.hab.habitat.HabReceiver;
import nl.sikken.bertrik.hab.habitat.HabitatUploader;
import nl.sikken.bertrik.hab.habitat.IHabitatRestApi;
import nl.sikken.bertrik.hab.habitat.Location;
import nl.sikken.bertrik.hab.ttn.MqttData;
import nl.sikken.bertrik.hab.ttn.MqttGateway;
import nl.sikken.bertrik.hab.ttn.TtnMessage;
import nl.sikken.bertrik.hab.ttn.TtnMessageGateway;
/**
* Bridge between the-things-network and the habhub network.
*
* Possible improvements:
* - put the MQTT functionality in its own module
* - add uncaught exception handler
* - add example systemd startup scripts
*/
public final class TtnHabBridge {
@ -111,6 +116,8 @@ public final class TtnHabBridge {
}
});
mqttClient.subscribe(config.getMqttTopic());
LOG.info("Started TTN-HAB bridge application");
}
private void handleMessageArrived(String topic, String message) {
@ -118,7 +125,7 @@ public final class TtnHabBridge {
try {
// try to decode the payload
final MqttData data = mapper.readValue(message, MqttData.class);
final TtnMessage data = mapper.readValue(message, TtnMessage.class);
final SodaqOnePayload sodaq = SodaqOnePayload.parse(data.getPayload());
LOG.info("Got SODAQ message: {}", sodaq);
@ -135,7 +142,7 @@ public final class TtnHabBridge {
// create listeners
final List<HabReceiver> receivers = new ArrayList<>();
for (MqttGateway gw : data.getMetaData().getMqttGateways()) {
for (TtnMessageGateway gw : data.getMetaData().getMqttGateways()) {
final HabReceiver receiver =
new HabReceiver(gw.getId(), new Location(gw.getLatitude(), gw.getLongitude(), gw.getAltitude()));
receivers.add(receiver);
@ -143,11 +150,11 @@ public final class TtnHabBridge {
// send listener data
for (HabReceiver receiver : receivers) {
uploader.uploadListenerData(receiver, now);
uploader.scheduleListenerDataUpload(receiver, now);
}
// send payload telemetry data
uploader.uploadPayloadTelemetry(line, receivers, now);
uploader.schedulePayloadTelemetryUpload(line, receivers, now);
} catch (IOException e) {
LOG.warn("JSON unmarshalling exception '{}' for {}", e.getMessage(), message);
} catch (BufferUnderflowException e) {
@ -165,7 +172,7 @@ public final class TtnHabBridge {
mqttClient.close();
} catch (MqttException e) {
// what can we do about this?
LOG.warn("Error closing MQTT client...");
LOG.warn("Error closing MQTT client: {}", e.getMessage());
}
uploader.stop();
}
@ -175,7 +182,7 @@ public final class TtnHabBridge {
try {
config.load(file);
} catch (IOException e) {
LOG.info("Failed to load config, attempt to write defaults...");
LOG.info("Failed to load config {}, writing defaults", file.getAbsoluteFile());
config.save(file);
}
return config;

Wyświetl plik

@ -20,7 +20,7 @@ public final class TtnHabBridgeConfig implements ITtnHabBridgeConfig {
HABITAT_URL("habitat.url", "http://habitat.habhub.org", "URL of the habitat server"),
HABITAT_TIMEOUT("habitat.timeout", "3000", "Timeout in milliseconds"),
MQTT_SERVER_URL("mqtt.serverurl", "eu.thethings.network", "URL to MQTT server"),
MQTT_SERVER_URL("mqtt.serverurl", "tcp://eu.thethings.network", "URL of the TTN MQTT server"),
MQTT_CLIENT_ID("mqtt.clientid", "ttnhabbridge", "MQTT client id"),
MQTT_USER_NAME("mqtt.username", "ttnmapper", "TTN application name used as MQTT user name"),
MQTT_USER_PASS("mqtt.password", "ttn-account-v2.Xc8BFRKeBK5nUhc9ikDcR-sbelgSMdHKnOQKMAiwpgI", "TTN application password"),
@ -41,7 +41,9 @@ public final class TtnHabBridgeConfig implements ITtnHabBridgeConfig {
private final Map<EConfigItem, String> props = new HashMap<>();
/**
* Create a configuration setting with defaults.
* Constructor.
*
* Configures all settings to their default value.
*/
public TtnHabBridgeConfig() {
for (EConfigItem e : EConfigItem.values()) {

Wyświetl plik

@ -1,9 +1,9 @@
package nl.sikken.bertrik.hab;
/**
* @author bertrik
* CRC16-CCITT implementation.
*/
public final class CcittCrc16 {
public final class CrcCcitt16 {
private static int[] crc_table = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129,

Wyświetl plik

@ -9,8 +9,7 @@ import java.util.Locale;
import java.util.TimeZone;
/**
* @author bertrik
*
* Representation of a HAB telemetry sentence.
*/
public final class Sentence {
@ -21,7 +20,7 @@ public final class Sentence {
private final double longitude;
private final double altitude;
private final CcittCrc16 crc16 = new CcittCrc16();
private final CrcCcitt16 crc16 = new CrcCcitt16();
private final List<String> extras = new ArrayList<>();
@ -54,6 +53,8 @@ public final class Sentence {
}
/**
* Formats the sentence into an ASCII string.
*
* @return a sentence formatted according to the basic UKHAS convention
*/
public String format() {

Wyświetl plik

@ -6,10 +6,11 @@ import java.nio.ByteOrder;
import java.util.Locale;
/**
* Encoding/decoding according to "SODAQ" format.
* Decoding according to "SODAQ" format, as used in (for example):
* https://github.com/SodaqMoja/SodaqOne-UniversalTracker
*/
public final class SodaqOnePayload {
private final long timeStamp;
private final double battVoltage;
private final int boardTemp;
@ -35,8 +36,8 @@ public final class SodaqOnePayload {
* @param numSats number of satellites used in fix
* @param ttf the time to fix
*/
public SodaqOnePayload(long timeStamp, double battVoltage, int boardTemp, double latitude, double longitude, double altitude,
double sog, int cog, int numSats, int ttf) {
public SodaqOnePayload(long timeStamp, double battVoltage, int boardTemp, double latitude, double longitude,
double altitude, double sog, int cog, int numSats, int ttf) {
this.timeStamp = timeStamp;
this.battVoltage = battVoltage;
this.boardTemp = boardTemp;
@ -58,8 +59,8 @@ public final class SodaqOnePayload {
*/
public static SodaqOnePayload parse(byte[] raw) throws BufferUnderflowException {
final ByteBuffer bb = ByteBuffer.wrap(raw).order(ByteOrder.LITTLE_ENDIAN);
final long timeStamp = (bb.getInt() & 0xFFFFFFFFL);
final double battVoltage = 3.0 + (bb.get() * 1.5 / 256);
final long time = (bb.getInt() & 0xFFFFFFFFL);
final double voltage = 3.0 + (bb.get() * 1.5 / 256);
final int boardTemp = bb.get();
final double latitude = bb.getInt() / 1e7;
final double longitude = bb.getInt() / 1e7;
@ -69,7 +70,7 @@ public final class SodaqOnePayload {
final int numSats = bb.get();
final int ttf = bb.get();
return new SodaqOnePayload(timeStamp, battVoltage, boardTemp, latitude, longitude, altitude, sog, cog, numSats, ttf);
return new SodaqOnePayload(time, voltage, boardTemp, latitude, longitude, altitude, sog, cog, numSats, ttf);
}
public long getTimeStamp() {
@ -111,11 +112,11 @@ public final class SodaqOnePayload {
public int getTtf() {
return ttf;
}
@Override
public String toString() {
return String.format(Locale.US, "ts=%d,batt=%.2f,lat=%f,lon=%f,alt=%.0f",
timeStamp, battVoltage, latitude, longitude, altitude);
return String.format(Locale.US, "ts=%d,batt=%.2f,temp=%d,lat=%f,lon=%f,alt=%.0f", timeStamp, battVoltage,
boardTemp, latitude, longitude, altitude);
}
}

Wyświetl plik

@ -29,24 +29,30 @@ import nl.sikken.bertrik.hab.habitat.docs.PayloadTelemetryDoc;
*
* Exchanges data with the habitat system.
* All actions run on a single thread for simplicity.
*
*/
public final class HabitatUploader {
private static final Logger LOG = LoggerFactory.getLogger(HabitatUploader.class);
private static MessageDigest sha256;
private final Logger LOG = LoggerFactory.getLogger(HabitatUploader.class);
private final ExecutorService executor = Executors.newSingleThreadExecutor();
private final Encoder base64Encoder = Base64.getEncoder();
private final MessageDigest sha256;
private final IHabitatRestApi restClient;
static {
try {
sha256 = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("No SHA-256 hash found");
}
/**
* Creates an actual REST client. Can be used in the constructor.
*
* @param url the URL to connect to
* @param timeout the connect and read timeout (ms)
* @return a new REST client
*/
public static IHabitatRestApi newRestClient(String url, int timeout) {
// create the REST client
LOG.info("Creating new habitat REST client with timeout {} for {}", timeout, url);
final WebTarget target = ClientBuilder.newClient().property(ClientProperties.CONNECT_TIMEOUT, timeout)
.property(ClientProperties.READ_TIMEOUT, timeout).target(url);
return WebResourceFactory.newResource(IHabitatRestApi.class, target);
}
/**
@ -55,6 +61,12 @@ public final class HabitatUploader {
* @param restClient the REST client used for uploading
*/
public HabitatUploader(IHabitatRestApi restClient) {
try {
this.sha256 = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
// this is fatal
throw new IllegalStateException("No SHA-256 hash found");
}
this.restClient = restClient;
}
@ -83,7 +95,7 @@ public final class HabitatUploader {
* @param receivers list of receivers that got this sentence
* @param date the current date
*/
public void uploadPayloadTelemetry(String sentence, List<HabReceiver> receivers, Date date) {
public void schedulePayloadTelemetryUpload(String sentence, List<HabReceiver> receivers, Date date) {
// encode sentence as raw bytes
final byte[] bytes = sentence.getBytes(StandardCharsets.US_ASCII);
@ -130,27 +142,13 @@ public final class HabitatUploader {
return DatatypeConverter.printHexBinary(hash).toLowerCase();
}
/**
* Creates an actual REST client. Can be used in the constructor.
*
* @param url the URL to connect to
* @param timeout the connect and read timeout (ms)
* @return a new REST client
*/
public static IHabitatRestApi newRestClient(String url, int timeout) {
// create the REST client
final WebTarget target = ClientBuilder.newClient().property(ClientProperties.CONNECT_TIMEOUT, timeout)
.property(ClientProperties.READ_TIMEOUT, timeout).target(url);
return WebResourceFactory.newResource(IHabitatRestApi.class, target);
}
/**
* Schedules new listener data to be sent to habitat.
*
* @param receiver the receiver data
* @param date the current date
*/
public void uploadListenerData(HabReceiver receiver, Date date) {
public void scheduleListenerDataUpload(HabReceiver receiver, Date date) {
executor.submit(() -> uploadListener(receiver, date));
}

Wyświetl plik

@ -16,10 +16,6 @@ public final class UploadResult {
@JsonProperty("rev")
private String rev;
private UploadResult() {
// jackson constructor
}
public boolean isOk() {
return ok;
}

Wyświetl plik

@ -1,7 +1,5 @@
package nl.sikken.bertrik.hab.habitat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -14,15 +12,6 @@ public final class UuidsList {
@JsonProperty("uuids")
private List<String> uuids;
private UuidsList() {
// jackson constructor
}
private UuidsList(Collection<String> uuids) {
this();
uuids = new ArrayList<>(uuids);
}
public List<String> getUuids() {
return uuids;

Wyświetl plik

@ -8,17 +8,18 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* @author bertrik
*
* Payload telemetry document.
*
* SEE http://habitat.habhub.org/jse/#schemas/payload_telemetry.json
*/
public final class PayloadTelemetryDoc {
private final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
private final String callSign;
private final byte[] rawBytes;
private final Date dateCreated;
private final Date dateUploaded;
private final String callSign;
private final byte[] rawBytes;
/**
* Constructor.
@ -28,10 +29,10 @@ public final class PayloadTelemetryDoc {
* @param rawBytes the raw telemetry string as bytes
*/
public PayloadTelemetryDoc(Date date, String callSign, byte[] rawBytes) {
this.callSign = callSign;
this.rawBytes = rawBytes;
this.dateCreated = date;
this.dateUploaded = date;
this.callSign = callSign;
this.rawBytes = rawBytes;
}
/**

Wyświetl plik

@ -7,7 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* Representation of a message received from the TTN MQTT stream.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public final class MqttData {
public final class TtnMessage {
@JsonProperty("app_id")
private String appId;
@ -28,7 +28,7 @@ public final class MqttData {
private byte[] payload;
@JsonProperty("metadata")
private MqttMetaData metaData;
private TtnMessageMetaData metaData;
public String getAppId() {
return appId;
@ -54,7 +54,7 @@ public final class MqttData {
return payload;
}
public MqttMetaData getMetaData() {
public TtnMessageMetaData getMetaData() {
return metaData;
}

Wyświetl plik

@ -7,7 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* Representation of a gateway in the metadata of the TTN MQTT JSON format.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public final class MqttGateway {
public final class TtnMessageGateway {
@JsonProperty("gtw_id")
private String id;

Wyświetl plik

@ -9,19 +9,19 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* Representation of meta-data part of MQTT message.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public final class MqttMetaData {
public final class TtnMessageMetaData {
@JsonProperty("time")
private String time;
@JsonProperty("gateways")
private List<MqttGateway> gateways;
private List<TtnMessageGateway> gateways;
public String getTime() {
return time;
}
public List<MqttGateway> getMqttGateways() {
public List<TtnMessageGateway> getMqttGateways() {
return gateways;
}

Wyświetl plik

@ -3,13 +3,18 @@ package nl.sikken.bertrik;
import java.io.File;
import java.io.IOException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/**
* Unit tests for TtnHabBridgeConfig.
*/
public final class TtnHabBridgeConfigTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
/**
* Verifies basic loading/saving of a configuration.
*
@ -18,7 +23,7 @@ public final class TtnHabBridgeConfigTest {
@Test
public void testLoadSave() throws IOException {
final TtnHabBridgeConfig config = new TtnHabBridgeConfig();
final File file = new File("test.properties");
final File file = new File(tempFolder.getRoot(), "test.properties");
config.save(file);
config.load(file);
}

Wyświetl plik

@ -0,0 +1,33 @@
package nl.sikken.bertrik.hab;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import org.junit.Assert;
import org.junit.Test;
/**
* Unit tests for CCITT CRC.
*/
public final class CrcCcitt16Test {
/**
* Verifies calculation of checksum
*
* Known good string with good CRC:
* $$hadie,181,10:42:10,54.422829,-6.741293,27799.3,1:10*002A
*
* @throws UnsupportedEncodingException
*/
@Test
public void testCrc() throws UnsupportedEncodingException {
final String s = "hadie,181,10:42:10,54.422829,-6.741293,27799.3,1:10";
final byte[] data = s.getBytes(StandardCharsets.US_ASCII);
final CrcCcitt16 crc = new CrcCcitt16();
int value = crc.calculate(data, 0xFFFF);
Assert.assertEquals(0x002A, value);
}
}

Wyświetl plik

@ -1,28 +1,31 @@
package nl.sikken.bertrik.hab;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import org.junit.Assert;
import org.junit.Test;
/**
* @author bertrik
*
* Unit tests of HAB telemetry sentence.
*/
public final class SentenceTest {
/**
* Verifies basic formatting.
*/
@Test
public void testSentence() throws UnsupportedEncodingException {
public void testSentence() {
final Sentence sentence = new Sentence("CALL", 1, new Date(0), 3.45, 6.78, 9.0);
final String s = sentence.format();
Assert.assertEquals("$$CALL,1,00:00:00,3.450000,6.780000,9.0*25E9\n", s);
}
/**
* Verifies that extra fields are formatted too.
*/
@Test
public void testSentenceExtras() throws UnsupportedEncodingException {
public void testSentenceExtras() {
final Sentence sentence = new Sentence("CALL", 1, new Date(), 3.45, 6.78, 9.0);
sentence.addField("hello");
final String s = sentence.format();
@ -30,23 +33,4 @@ public final class SentenceTest {
Assert.assertTrue(s.contains("hello"));
}
/**
* Verifies calculation of checksum
*
* Known good string with good CRC:
* $$hadie,181,10:42:10,54.422829,-6.741293,27799.3,1:10*002A
*
* @throws UnsupportedEncodingException
*/
@Test
public void testCrc() throws UnsupportedEncodingException {
final String s = "hadie,181,10:42:10,54.422829,-6.741293,27799.3,1:10";
final byte[] data = s.getBytes(StandardCharsets.US_ASCII);
final CcittCrc16 crc = new CcittCrc16();
int value = crc.calculate(data, 0xFFFF);
Assert.assertEquals(0x002A, value);
}
}

Wyświetl plik

@ -18,23 +18,21 @@ public final class HabitatUploaderTest {
* Happy flow scenario.
*
* Verifies that a request to upload results in an actual upload.
*
* @throws InterruptedException
*/
@Test
public void testMockedHappyFlow() throws InterruptedException {
public void testMockedHappyFlow() {
// create a mocked rest client
final IHabitatRestApi restClient = Mockito.mock(IHabitatRestApi.class);
Mockito.when(restClient.updateListener(Mockito.anyString(), Mockito.anyString())).thenReturn("OK");
final HabitatUploader uploader = new HabitatUploader(restClient);
// test it
// verify upload using the uploader
uploader.start();
try {
final HabReceiver receiver = new HabReceiver("BERTRIK", null);
final Date date = new Date();
final Sentence sentence = new Sentence("NOTAFLIGHT", 1, date, 52.0182307, 4.695772, 1000);
uploader.uploadPayloadTelemetry(sentence.format(), Arrays.asList(receiver), date);
uploader.schedulePayloadTelemetryUpload(sentence.format(), Arrays.asList(receiver), date);
Mockito.verify(restClient, Mockito.timeout(3000)).updateListener(Mockito.anyString(), Mockito.anyString());
} finally {
@ -42,6 +40,11 @@ public final class HabitatUploaderTest {
}
}
/**
* Verifies upload of payload telemetry to the actual habitat server on the internet.
*
* @throws InterruptedException in case the sleep got interrupted
*/
@Test
@Ignore("this is not a junit test")
public void testActualPayloadUpload() throws InterruptedException {
@ -52,7 +55,7 @@ public final class HabitatUploaderTest {
final Date date = new Date();
final Sentence sentence = new Sentence("NOTAFLIGHT", 1, date, 52.0182307, 4.695772, 1000);
final HabReceiver receiver = new HabReceiver("BERTRIK", null);
uploader.uploadPayloadTelemetry(sentence.format(), Arrays.asList(receiver), date);
uploader.schedulePayloadTelemetryUpload(sentence.format(), Arrays.asList(receiver), date);
Thread.sleep(3000);
} finally {
uploader.stop();
@ -60,8 +63,9 @@ public final class HabitatUploaderTest {
}
/**
* Verifies upload of listener information and telemetry.
* @throws InterruptedException
* Verifies upload of listener information and telemetry to the actual habitat server on the internet.
*
* @throws InterruptedException in case the sleep got interrupted
*/
@Test
@Ignore("this is not a junit test")
@ -71,7 +75,7 @@ public final class HabitatUploaderTest {
try {
final Date date = new Date();
final HabReceiver receiver = new HabReceiver("BERTRIK", new Location(52.0182307, 4.695772, 15));
uploader.uploadListenerData(receiver, date);
uploader.scheduleListenerDataUpload(receiver, date);
Thread.sleep(3000);
} finally {
uploader.stop();

Wyświetl plik

@ -12,9 +12,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import nl.sikken.bertrik.hab.SodaqOnePayload;
/**
* Unit tests for MqttData.
* Unit tests for TTN messages.
*/
public final class MqttDataTest {
public final class TtnMessageTest {
private static final String DATA = "{\"app_id\":\"ttnmapper\",\"dev_id\":\"mapper2\","
+ "\"hardware_serial\":\"0004A30B001ADBC5\",\"port\":1,\"counter\":4,"
@ -35,7 +35,7 @@ public final class MqttDataTest {
@Test
public void testDecode() throws JsonParseException, JsonMappingException, IOException {
final ObjectMapper mapper = new ObjectMapper();
final MqttData mqttData = mapper.readValue(DATA, MqttData.class);
final TtnMessage mqttData = mapper.readValue(DATA, TtnMessage.class);
final byte[] raw = mqttData.getPayload();
// check gateway field

Wyświetl plik

@ -0,0 +1,21 @@
# URL of the habitat server
habitat.url=http://habitat.habhub.org
# Timeout in milliseconds
habitat.timeout=3000
# URL of the TTN MQTT server
mqtt.serverurl=tcp://eu.thethings.network
# MQTT client id
mqtt.clientid=ttnhabbridge
# TTN application name used as MQTT user name
mqtt.username=ttnmapper
# TTN application password
mqtt.password=ttn-account-v2.Xc8BFRKeBK5nUhc9ikDcR-sbelgSMdHKnOQKMAiwpgI
# MQTT topic to subscribe to
mqtt.topic=ttnmapper/devices/+/up