From 72bcaebdb888f381df24e0c633cdb3657ba6c6bb Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 25 Aug 2020 19:20:04 +0100 Subject: [PATCH] Fix CtyXml decoding issue. Add multicast support for WSJT-X. Switch to WsjtxUdpLib. --- .gitignore | 4 +- ft8spotter.tests/DecodeMessageTests.cs | 76 --------- ft8spotter/CtyXml.cs | 17 +- ft8spotter/DecodeMesssage.cs | 214 ------------------------- ft8spotter/Program.cs | 203 ++++++++++------------- ft8spotter/ft8spotter.csproj | 4 + 6 files changed, 107 insertions(+), 411 deletions(-) delete mode 100644 ft8spotter.tests/DecodeMessageTests.cs delete mode 100644 ft8spotter/DecodeMesssage.cs diff --git a/.gitignore b/.gitignore index 4ce6fdd..293edee 100644 --- a/.gitignore +++ b/.gitignore @@ -337,4 +337,6 @@ ASALocalRun/ .localhistory/ # BeatPulse healthcheck temp database -healthchecksdb \ No newline at end of file +healthchecksdb +/ft8spotter/Properties/launchSettings.json +/ft8spotter/cty.xml.bak diff --git a/ft8spotter.tests/DecodeMessageTests.cs b/ft8spotter.tests/DecodeMessageTests.cs deleted file mode 100644 index 8b6de92..0000000 --- a/ft8spotter.tests/DecodeMessageTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text; -using Xunit; - -namespace ft8spotter.tests -{ - public class DecodeMessageTests - { - // 081700 -12 0.3 367 ~ EC1AIJ US2YW KN28 - string testData = @"ad bc cb da 00 00 00 02 -00 00 00 02 00 00 00 06 -57 53 4a 54 2d 58 01 01 -c7 04 60 ff ff ff f4 3f -d3 33 33 40 00 00 00 00 -00 01 6f 00 00 00 01 7e -00 00 00 11 45 43 31 41 -49 4a 20 55 53 32 59 57 -20 4b 4e 32 38 00 00"; - - /* - * ad bc cb da = magic number - * 00 00 00 02 = schema version - * 00 00 00 02 = message type 2 (decode) - * 00 00 00 06 = Id field: next 6 bytes are a string - * 57 53 4a 54 2d 58 = Id field: WSJT-X - * 01 = new field - * 01 c7 04 60 = 29,820,000ms since midnight - * ff ff ff f4 = snr -12 (big endian int32 - https://www.scadacore.com/tools/programming-calculators/online-hex-converter/) - * 3f d3 33 33 40 00 00 00 - delta time (double) - * 00 00 01 6f - delta frequency: 367 - * 00 00 00 01 = mode field: next 1 byte is a string - * 7e = mode field = ~ - * 00 00 00 11 = message field: next 0x11 / 17 bytes are a string - * 45 43 31 41 49 4a 20 55 53 32 59 57 20 4b 4e 32 38 = EC1AIJ US2YW KN28 - * 00 = low confidence - * 00 = off air - */ - - IEnumerable message - { - get - { - foreach (string line in testData.Split(Environment.NewLine)) - { - foreach (string hexString in line.Split(' ', StringSplitOptions.RemoveEmptyEntries)) - { - yield return byte.Parse(hexString, NumberStyles.HexNumber); - } - } - } - } - - [Fact] - public void SchemaVersion() - { - var data = message.ToArray(); - - Assert.Equal(ParseResult.Success, DecodeMessage.TryParse(data, out DecodeMessage decodeMessage)); - - Assert.Equal(2, decodeMessage.SchemaVersion); - Assert.Equal("WSJT-X", decodeMessage.Id); - Assert.True(decodeMessage.New); - Assert.Equal(TimeSpan.FromSeconds(29820), decodeMessage.SinceMidnight); - Assert.Equal(-12, decodeMessage.Snr); - Assert.Equal(0.3, Math.Round(decodeMessage.DeltaTime, 1)); - Assert.Equal(367, decodeMessage.DeltaFrequency); - Assert.Equal("~", decodeMessage.Mode); - Assert.Equal("EC1AIJ US2YW KN28", decodeMessage.Message); - Assert.False(decodeMessage.LowConfidence); - Assert.False(decodeMessage.OffAir); - } - } -} \ No newline at end of file diff --git a/ft8spotter/CtyXml.cs b/ft8spotter/CtyXml.cs index b884740..1f690a0 100644 --- a/ft8spotter/CtyXml.cs +++ b/ft8spotter/CtyXml.cs @@ -8,7 +8,7 @@ namespace ft8spotter { public class ClublogCtyXml { - internal const string ClublogNamespace = "https://clublog.org/cty/v1.2"; + //internal const string ClublogNamespace = "https://clublog.org/cty/v1.2"; public DateTime Updated { get; set; } public Entity[] Entities { get; set; } @@ -33,9 +33,16 @@ namespace ft8spotter return result; } - private static T[] Fetch(XDocument xDocument, string parent, string elementName, Func parse) => xDocument.Element(XName.Get("clublog", ClublogNamespace)) - .Descendants(XName.Get(parent, ClublogNamespace)) - .Descendants(XName.Get(elementName, ClublogNamespace)) + static string GetNamespace(XDocument doc) + { + var ns = doc.Root.GetDefaultNamespace()?.NamespaceName; + + return ns; + } + + private static T[] Fetch(XDocument xDocument, string parent, string elementName, Func parse) => xDocument.Element(XName.Get("clublog", GetNamespace(xDocument))) + .Descendants(XName.Get(parent, GetNamespace(xDocument))) + .Descendants(XName.Get(elementName, GetNamespace(xDocument))) .Select(parse) .ToArray(); @@ -85,7 +92,7 @@ namespace ft8spotter internal static string GetString(XElement xe, string elementName) { - return xe.Element(XName.Get(elementName, ClublogCtyXml.ClublogNamespace))?.Value; + return xe.Element(XName.Get(elementName, GetNamespace(xe.Document)))?.Value; } internal static int? GetNullableInt(XElement xe, string v) diff --git a/ft8spotter/DecodeMesssage.cs b/ft8spotter/DecodeMesssage.cs deleted file mode 100644 index 84a3bd7..0000000 --- a/ft8spotter/DecodeMesssage.cs +++ /dev/null @@ -1,214 +0,0 @@ -using System; -using System.Linq; -using System.Net; - -namespace ft8spotter -{ - /// - /// A .NET type which parses the format of UDP datagrams emitted from WSJT-X on UDP port 2237, - /// for the Decode message type (the type emitted when WSJT-X decodes an FT8 frame) - /// - public class DecodeMessage - { - /* - * Excerpt from NetworkMessage.hpp in WSJT-X source code: - * - * WSJT-X Message Formats - * ====================== - * - * All messages are written or read using the QDataStream derivatives - * defined below, note that we are using the default for floating - * point precision which means all are double precision i.e. 64-bit - * IEEE format. - * - * Message is big endian format - * - * Header format: - * - * 32-bit unsigned integer magic number 0xadbccbda - * 32-bit unsigned integer schema number - * - * Payload format: - * - * As per the QDataStream format, see below for version used and - * here: - * - * http://doc.qt.io/qt-5/datastreamformat.html - * - * for the serialization details for each type, at the time of - * writing the above document is for Qt_5_0 format which is buggy - * so we use Qt_5_4 format, differences are: - * - * QDateTime: - * QDate qint64 Julian day number - * QTime quint32 Milli-seconds since midnight - * timespec quint8 0=local, 1=UTC, 2=Offset from UTC - * (seconds) - * 3=time zone - * offset qint32 only present if timespec=2 - * timezone several-fields only present if timespec=3 - * - * we will avoid using QDateTime fields with time zones for simplicity. - * - * Type utf8 is a utf-8 byte string formatted as a QByteArray for - * serialization purposes (currently a quint32 size followed by size - * bytes, no terminator is present or counted). - * - * The QDataStream format document linked above is not complete for - * the QByteArray serialization format, it is similar to the QString - * serialization format in that it differentiates between empty - * strings and null strings. Empty strings have a length of zero - * whereas null strings have a length field of 0xffffffff. - * - * Decode Out 2 quint32 4 bytes? - * Id (unique key) utf8 4 bytes, that number of chars, no terminator - * New bool 1 byte or bit? - * Time QTime quint32 Milliseconds since midnight (4 bytes?) - * snr qint32 4 bytes? - * Delta time (S) float (serialized as double) 8 bytes - * Delta frequency (Hz) quint32 4 bytes - * Mode utf8 4 bytes, that number of chars, no terminator - * Message utf8 4 bytes, that number of chars, no terminator - * Low confidence bool 1 byte or bit? - * Off air bool 1 byte or bit? - * - * The decode message is sent when a new decode is completed, in - * this case the 'New' field is true. It is also used in response - * to a "Replay" message where each old decode in the "Band - * activity" window, that has not been erased, is sent in order - * as a one of these messages with the 'New' field set to false. - * See the "Replay" message below for details of usage. Low - * confidence decodes are flagged in protocols where the decoder - * has knows that a decode has a higher than normal probability - * of being false, they should not be reported on publicly - * accessible services without some attached warning or further - * validation. Off air decodes are those that result from playing - * back a .WAV file. - * - * From MessageServer.cpp: - - case NetworkMessage::Decode: - { - // unpack message - bool is_new {true}; - QTime time; - qint32 snr; - float delta_time; - quint32 delta_frequency; - QByteArray mode; - QByteArray message; - bool low_confidence {false}; - bool off_air {false}; - in >> is_new >> time >> snr >> delta_time >> delta_frequency >> mode >> message >> low_confidence >> off_air; - if (check_status (in) != Fail) - { - Q_EMIT self_->decode (is_new, id, time, snr, delta_time, delta_frequency - , QString::fromUtf8 (mode), QString::fromUtf8 (message) - , low_confidence, off_air); - } - } - break; - * - */ - - public int SchemaVersion { get; set; } - public string Id { get; set; } - public bool New { get; set; } - public TimeSpan SinceMidnight { get; set; } - public int Snr { get; set; } - public double DeltaTime { get; set; } - public int DeltaFrequency { get; set; } - public string Mode { get; set; } - public string Message { get; set; } - public bool LowConfidence { get; set; } - public bool OffAir { get; set; } - - private const int DECODE_MESSAGE_TYPE = 2; - - public static ParseResult TryParse(byte[] message, out DecodeMessage decodeMessage) - { - if (!Enumerable.SequenceEqual(message.Take(4), new byte[] { 0xad, 0xbc, 0xcb, 0xda })) - { - decodeMessage = null; - return ParseResult.InvalidMagicNumber; - } - - decodeMessage = new DecodeMessage(); - - int cur = 4; // length of magic number - decodeMessage.SchemaVersion = GetInt32(message, ref cur); - - if (GetInt32(message, ref cur) != DECODE_MESSAGE_TYPE) - { - return ParseResult.NotADecodeMessage; - } - - decodeMessage.Id = GetString(message, ref cur); - decodeMessage.New = GetBool(message, ref cur); - decodeMessage.SinceMidnight = TimeSpan.FromMilliseconds(GetInt32(message, ref cur)); - decodeMessage.Snr = GetInt32(message, ref cur); - decodeMessage.DeltaTime = GetDouble(message, ref cur); - decodeMessage.DeltaFrequency = GetInt32(message, ref cur); - decodeMessage.Mode = GetString(message, ref cur); - decodeMessage.Message = GetString(message, ref cur); - decodeMessage.LowConfidence = GetBool(message, ref cur); - decodeMessage.OffAir = GetBool(message, ref cur); - - return ParseResult.Success; - } - - private static int GetInt32(byte[] message, ref int cur) - { - int result = IPAddress.NetworkToHostOrder(BitConverter.ToInt32(message, cur)); - cur += sizeof(int); - return result; - } - - private static double GetDouble(byte[] message, ref int cur) - { - double result; - if (BitConverter.IsLittleEndian) - { - // x64 - result = BitConverter.ToDouble(message.Skip(cur).Take(sizeof(double)).Reverse().ToArray(), 0); - } - else - { - // who knows what - result = BitConverter.ToDouble(message, cur); - } - - cur += sizeof(double); - return result; - } - - private static bool GetBool(byte[] message, ref int cur) - { - bool result = message[cur] != 0; - cur += sizeof(bool); - return result; - } - - private static string GetString(byte[] message, ref int cur) - { - int numBytesInField = GetInt32(message, ref cur); - - char[] letters = new char[numBytesInField]; - for (int i = 0; i < numBytesInField; i++) - { - letters[i] = (char)message[cur + i]; - } - - cur += numBytesInField; - - return new string(letters); - } - } - - public enum ParseResult - { - Success, - NotADecodeMessage, - InvalidMagicNumber - } -} \ No newline at end of file diff --git a/ft8spotter/Program.cs b/ft8spotter/Program.cs index f347fe3..5dcc60f 100644 --- a/ft8spotter/Program.cs +++ b/ft8spotter/Program.cs @@ -1,13 +1,13 @@ -using System; +using M0LTE.WsjtxUdpLib.Client; +using M0LTE.WsjtxUdpLib.Messages.Out; +using System; using System.Collections.Generic; -using System.Data; using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; -using System.Net.Sockets; using System.Text; using System.Threading; @@ -21,6 +21,10 @@ namespace ft8spotter static ClublogCtyXml ctyXml; static void Main(string[] args) { + httpClient.DefaultRequestHeaders.UserAgent.TryParseAdd("ft8spotter"); + + bool multicast = GetAndConsumeArg(args, "--multicast"); + bool all = args.Any(a => a == "--all"); bool grids = args.Any(a => a == "--grids"); @@ -98,90 +102,100 @@ it is, it highlights the call in red in the console window."); { File.Delete("cty.xml"); File.Move("cty.xml.bak", "cty.xml"); - Console.WriteLine("Failed to update cty.xml"); + Console.WriteLine($"Failed to update cty.xml: {ex.Message}"); ctyXml = ClublogCtyXml.Parse(File.ReadAllText("cty.xml")); } } - const int port = 2237; - using (var client = new UdpClient(port, AddressFamily.InterNetwork)) + WsjtxClient wsjsxClient; + if (multicast) { - Console.WriteLine($"Listening for WSJT-X on UDP port {port}"); + //TODO: make hard-coded group an argument + wsjsxClient = new WsjtxClient(Callback, IPAddress.Parse("239.1.2.3"), multicast: true); + } + else + { + wsjsxClient = new WsjtxClient(Callback, IPAddress.Loopback); + } + + Console.WriteLine($"Listening for WSJT-X"); + + Thread.CurrentThread.Join(); + + void Callback(WsjtxMessage wsjtxMessage) + { + if (!(wsjtxMessage is DecodeMessage decodeMessage)) + { + return; + } var sw = Stopwatch.StartNew(); - while (true) + string heardCall = GetHeardCall(decodeMessage.Message); + + if (heardCall == null) { - var ipep = new IPEndPoint(IPAddress.Loopback, 0); + return; + } - byte[] msg = client.Receive(ref ipep); + var entity = GetEntity(heardCall); - if (ParseResult.Success != DecodeMessage.TryParse(msg, out DecodeMessage decodeMessage)) - continue; + string grid = GetGrid(decodeMessage.Datagram); + var needed = entity == null ? Needed.No : GetNeeded(band, entity.Adif, grids ? grid : null, "ft8"); - - //if (msg[11] == 0x02) - //{ - //string heardCall = GetHeardCall(msg); - - string heardCall = GetHeardCall(decodeMessage.Message); - - if (heardCall == null) - continue; - - var entity = GetEntity(heardCall); - - string grid = GetGrid(msg); - - var needed = entity == null ? Needed.No : GetNeeded(band, entity.Adif, grids ? grid : null, "ft8"); - - if (all || !Needed.No.Equals(needed)) + if (all || !Needed.No.Equals(needed)) + { + if (sw.Elapsed > TimeSpan.FromSeconds(5)) { - if (sw.Elapsed > TimeSpan.FromSeconds(5)) - { - Console.WriteLine($"--- {DateTime.Now:HH:mm:ss} --------------------------"); - sw.Restart(); - } - - var colBefore = Console.ForegroundColor; - if (needed.NewCountryOnAnyBand) - { - Console.ForegroundColor = ConsoleColor.Green; - } - else if (needed.NewCountryOnBand) - { - Console.ForegroundColor = ConsoleColor.Yellow; - } - else if (needed.NewCountryOnBandOnMode) - { - Console.ForegroundColor = ConsoleColor.DarkYellow; - } - else if (needed.NewGridOnAnyBand) - { - Console.ForegroundColor = ConsoleColor.Red; - } - else if (needed.NewGridOnBand) - { - Console.ForegroundColor = ConsoleColor.Magenta; - } - else if (needed.NewGridOnBandOnMode) - { - Console.ForegroundColor = ConsoleColor.DarkRed; - } - WriteAtColumn(0, needed, 19); - WriteAtColumn(19, heardCall, 10); - WriteAtColumn(30, decodeMessage.Snr, 4); - WriteAtColumn(34, IsGrid(grid) ? grid : String.Empty, 4); - WriteAtColumn(39, entity?.Adif, 3); - WriteAtColumn(43, (entity?.Entity) ?? "Unknown", 50); - - Console.WriteLine(); - Console.ForegroundColor = colBefore; + Console.WriteLine($"--- {DateTime.Now:HH:mm:ss} --------------------------"); + sw.Restart(); } + + var colBefore = Console.ForegroundColor; + if (needed.NewCountryOnAnyBand) + { + Console.ForegroundColor = ConsoleColor.Green; + } + else if (needed.NewCountryOnBand) + { + Console.ForegroundColor = ConsoleColor.Yellow; + } + else if (needed.NewCountryOnBandOnMode) + { + Console.ForegroundColor = ConsoleColor.DarkYellow; + } + else if (needed.NewGridOnAnyBand) + { + Console.ForegroundColor = ConsoleColor.Red; + } + else if (needed.NewGridOnBand) + { + Console.ForegroundColor = ConsoleColor.Magenta; + } + else if (needed.NewGridOnBandOnMode) + { + Console.ForegroundColor = ConsoleColor.DarkRed; + } + WriteAtColumn(0, needed, 19); + WriteAtColumn(19, heardCall, 10); + WriteAtColumn(30, decodeMessage.Snr, 4); + WriteAtColumn(34, IsGrid(grid) ? grid : String.Empty, 4); + WriteAtColumn(39, entity?.Adif, 3); + WriteAtColumn(43, (entity?.Entity) ?? "Unknown", 50); + + Console.WriteLine(); + Console.ForegroundColor = colBefore; } } } + private static bool GetAndConsumeArg(string[] args, string v) + { + bool present = args.Contains(v); + args = args.Where(a => a != v).ToArray(); + return present; + } + private static string GetHeardCall(string text) { string[] split = text.Split(' '); @@ -231,7 +245,11 @@ it is, it highlights the call in red in the console window."); else { string str = heardCall.ToString(); - if (str.Length <= max) + if (str == null) + { + toWrite = ""; + } + else if (str.Length <= max) { toWrite = str; } @@ -243,51 +261,6 @@ it is, it highlights the call in red in the console window."); Console.Write(toWrite); } - private static int GetSnr(byte[] msg, string call) - { - Console.WriteLine(call); - - int cur = 0; - foreach (var batch in msg.Batch(8)) - { - Console.Write(cur.ToString("00") + " "); - foreach (var b in batch) - { - string bytestr = b.ToString("X").ToLower(); - - if (bytestr.Length == 1) - { - bytestr = "0" + bytestr; - } - - Console.Write(bytestr + " "); - } - Console.WriteLine(); - - Console.Write(" "); - foreach (var b in batch) - { - char ch = (char)b; - if (Char.IsLetterOrDigit(ch) || Char.IsPunctuation(ch) || Char.IsSymbol(ch) || (ch == ' ')) - { - Console.Write(ch); - Console.Write(" "); - } - else if (ch == 0) - { - Console.Write(" "); - } - } - Console.WriteLine(); - Console.WriteLine(); - cur += 8; - } - - Debugger.Break(); - - return 0; - } - private static string GetGrid(byte[] msg) { string text; diff --git a/ft8spotter/ft8spotter.csproj b/ft8spotter/ft8spotter.csproj index 489b660..599ea0e 100644 --- a/ft8spotter/ft8spotter.csproj +++ b/ft8spotter/ft8spotter.csproj @@ -15,4 +15,8 @@ + + + +