mirror
/
radiosonde_auto_rx
kopia lustrzana https://github.com/projecthorus/radiosonde_auto_rx
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Relax Vaisala callsign regex, add option to log raw data to file (mostly untested)

pull/517/head
Mark Jessop 2021-06-03 20:40:18 +09:30
rodzic 663b95495a
commit bb4d7b0065
7 zmienionych plików z 119 dodań i 81 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
auto_rx/auto_rx.py
Wyświetl plik

@ -245,9 +245,9 @@ def start_decoder(freq, sonde_type):
timeout=config["rx_timeout"],
telem_filter=telemetry_filter,
rs92_ephemeris=rs92_ephemeris,
imet_location=config["station_code"],
rs41_drift_tweak=config["rs41_drift_tweak"],
experimental_decoder=config["experimental_decoders"][_exp_sonde_type],
save_raw_hex=config["save_raw_hex"]
)
autorx.sdr_list[_device_idx]["task"] = autorx.task_list[freq]["task"]
@ -566,7 +566,9 @@ def telemetry_filter(telemetry):
# This will need to be re-evaluated if we're still using this code in 2021!
# UPDATE: Had some confirmation that Vaisala will continue to use the alphanumeric numbering up until
# ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
vaisala_callsign_valid = re.match(r"[E-Z][0-5][\d][1-7]\d{4}", _serial)
# Modified 2021-06 to be more flexible and match older sondes, and reprogrammed sondes.
# Still needs a letter at the start, but the numbers don't need to match the format exactly.
vaisala_callsign_valid = re.match(r"[C-Z][\d][\d][\d]\d{4}", _serial)
# Just make sure we're not getting the 'xxxxxxxx' unknown serial from the DFM decoder.
if "DFM" in telemetry["type"]:

2
auto_rx/autorx/__init__.py
Wyświetl plik

@ -17,7 +17,7 @@ except ImportError:
# MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
# PATCH - Small changes, or minor feature additions.
__version__ = "1.5.3"
__version__ = "1.5.4-beta1"
# Global Variables

11
auto_rx/autorx/config.py
Wyświetl plik

@ -161,6 +161,7 @@ def read_auto_rx_config(filename, no_sdr_test=False):
"save_detection_audio": False,
"save_decode_audio": False,
"save_decode_iq": False,
"save_raw_hex": False,
# URL for the Habitat DB Server.
# As of July 2018 we send via sondehub.org, which will allow us to eventually transition away
# from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists.
@ -614,6 +615,16 @@ def read_auto_rx_config(filename, no_sdr_test=False):
)
auto_rx_config["web_control"] = False
auto_rx_config["web_password"] = "none"
try:
auto_rx_config["save_raw_hex"] = config.getboolean(
"debugging", "save_raw_hex"
)
except:
logging.warning(
"Config - Did not find save_raw_hex setting, using default (disabled)"
)
auto_rx_config["save_raw_hex"] = False
# If we are being called as part of a unit test, just return the config now.
if no_sdr_test:

64
auto_rx/autorx/decode.py
Wyświetl plik

@ -6,9 +6,11 @@
# Released under GNU GPL v3 or later
#
import autorx
import datetime
import logging
import json
import os
import os.path
import signal
import subprocess
import time
@ -133,7 +135,7 @@ class SondeDecoder(object):
rs92_ephemeris=None,
rs41_drift_tweak=False,
experimental_decoder=False,
imet_location="SONDE",
save_raw_hex=False
):
""" Initialise and start a Sonde Decoder.
@ -163,8 +165,7 @@ class SondeDecoder(object):
rs41_drift_tweak (bool): If True, add a high-pass filter in the decode chain, which can improve decode performance on drifty SDRs.
experimental_decoder (bool): If True, use the experimental fsk_demod-based decode chain.
imet_location (str): OPTIONAL - A location field which is use in the generation of iMet unique ID.
save_raw_hex (bool): If True, save the raw hex output from the decoder to a file.
"""
# Thread running flag
self.decoder_running = True
@ -187,7 +188,16 @@ class SondeDecoder(object):
self.rs92_ephemeris = rs92_ephemeris
self.rs41_drift_tweak = rs41_drift_tweak
self.experimental_decoder = experimental_decoder
self.imet_location = imet_location
self.save_raw_hex = save_raw_hex
self.raw_file = None
# Raw hex filename
if self.save_raw_hex:
_outfilename = f"{datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')}_{self.sonde_type}_{int(self.sonde_freq)}.raw"
_outfilename = os.path.join(autorx.logging_path, _outfilename)
self.raw_file_option = "-r"
else:
self.raw_file_option = ""
# iMet ID store. We latch in the first iMet ID we calculate, to avoid issues with iMet-1-RS units
# which don't necessarily have a consistent packet count to time increment ratio.
@ -281,6 +291,12 @@ class SondeDecoder(object):
self.log_error("Could not generate decoder command. Not starting decoder.")
self.decoder_running = False
else:
if self.save_raw_hex:
self.log_debug(f"Opening {_outfilename} to save decoder raw data.")
# Open the log file in binary mode.
self.raw_file = open(_outfilename, 'wb')
# Start up the decoder thread.
self.decode_process = None
self.async_reader = None
@ -467,7 +483,7 @@ class SondeDecoder(object):
decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
# iMet-4 (IMET1RS) decoder
decode_cmd += "./imet1rs_dft --json 2>/dev/null"
decode_cmd += f"./imet1rs_dft --json {self.raw_file_option} 2>/dev/null"
elif self.sonde_type == "IMET5":
# iMet-54 Sondes
@ -487,7 +503,7 @@ class SondeDecoder(object):
# iMet-54 Decoder
decode_cmd += (
"./imet54mod --ecc --IQ 0.0 --lp - 48000 16 --json --ptu 2>/dev/null"
f"./imet54mod --ecc --IQ 0.0 --lp - 48000 16 --json --ptu {self.raw_file_option} 2>/dev/null"
)
elif self.sonde_type == "MRZ":
@ -530,7 +546,7 @@ class SondeDecoder(object):
if self.save_decode_audio:
decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
# iMet-4 (IMET1RS) decoder
# LMS6-1680 decoder
if self.inverted:
self.log_debug("Using inverted MK2A decoder.")
decode_cmd += "./mk2a_lms1680 -i --json 2>/dev/null"
@ -588,7 +604,7 @@ class SondeDecoder(object):
decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
# Meisei IMS-100 decoder
decode_cmd += "./meisei100mod --json 2>/dev/null"
decode_cmd += f"./meisei100mod --json {self.raw_file_option} 2>/dev/null"
elif self.sonde_type == "UDP":
# UDP Input Mode.
@ -655,7 +671,7 @@ class SondeDecoder(object):
_baud_rate,
)
decode_cmd = "./rs41mod --ptu2 --json --softin -i 2>/dev/null"
decode_cmd = f"./rs41mod --ptu2 --json --softin -i {self.raw_file_option} 2>/dev/null"
# RS41s transmit pulsed beacons - average over the last 2 frames, and use a peak-hold
demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@ -731,8 +747,8 @@ class SondeDecoder(object):
)
decode_cmd = (
"./rs92mod -vx -v --crc --ecc --vel --json --softin -i %s %s 2>/dev/null"
% (_rs92_gps_data, _ptu_ops)
"./rs92mod -vx -v --crc --ecc --vel --json --softin -i %s %s %s 2>/dev/null"
% (_rs92_gps_data, _ptu_ops, self.raw_file_option)
)
# RS92s transmit continuously - average over the last 2 frames, and use a mean
@ -775,7 +791,7 @@ class SondeDecoder(object):
# DFM decoder
decode_cmd = (
"./dfm09mod -vv --ecc --json --dist --auto --softin -i 2>/dev/null"
f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option} 2>/dev/null"
)
# DFM sondes transmit continuously - average over the last 2 frames, and peak hold
@ -815,7 +831,7 @@ class SondeDecoder(object):
)
# M10 decoder
decode_cmd = "./m10mod --json --ptu -vvv --softin -i 2>/dev/null"
decode_cmd = f"./m10mod --json --ptu -vvv --softin -i {self.raw_file_option} 2>/dev/null"
# M10 sondes transmit in short, irregular pulses - average over the last 2 frames, and use a peak hold
demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@ -853,7 +869,7 @@ class SondeDecoder(object):
)
# M20 decoder
decode_cmd = "./mXXmod --json --ptu -vvv --softin -i 2>/dev/null"
decode_cmd = f"./mXXmod --json --ptu -vvv --softin -i {self.raw_file_option} 2>/dev/null"
# M20 sondes transmit in short, irregular pulses - average over the last 2 frames, and use a peak hold
demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@ -892,7 +908,7 @@ class SondeDecoder(object):
_baud_rate,
)
decode_cmd = "./lms6Xmod --json --softin --vit2 -i 2>/dev/null"
decode_cmd = f"./lms6Xmod --json --softin --vit2 -i {self.raw_file_option} 2>/dev/null"
# LMS sondes transmit continuously - average over the last 2 frames, and use a peak hold
demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@ -931,7 +947,7 @@ class SondeDecoder(object):
_baud_rate,
)
decode_cmd = "./imet54mod --ecc --json --softin -i --ptu 2>/dev/null"
decode_cmd = f"./imet54mod --ecc --json --softin -i --ptu {self.raw_file_option} 2>/dev/null"
# iMet54 sondes transmit in bursts. Use a peak hold.
demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@ -972,7 +988,7 @@ class SondeDecoder(object):
)
# MRZ decoder
decode_cmd = "./mp3h1mod --auto --json --softin --ptu 2>/dev/null"
decode_cmd = f"./mp3h1mod --auto --json --softin --ptu {self.raw_file_option} 2>/dev/null"
# MRZ sondes transmit continuously - average over the last frame, and use a peak hold
demod_stats = FSKDemodStats(averaging_time=1.0, peak_hold=True)
@ -1126,9 +1142,14 @@ class SondeDecoder(object):
# Don't even try and decode lines which don't start with a '{'
# These may be other output from the decoder, which we shouldn't try to parse.
# TODO: Perhaps we should add the option to log the raw data output from the decoders?
# If we have raw logging enabled, log these lines to disk.
if data.decode("ascii")[0] != "{":
return
# Save the line verbatim to the raw data file, if we have that enabled
if self.raw_file:
self.raw_file.write(data)
else:
return
else:
try:
@ -1255,7 +1276,7 @@ class SondeDecoder(object):
# Generate a unique ID based on the power-on time and frequency, as iMet sondes don't send one.
# Latch this ID and re-use it for the entire decode run.
if self.imet_id == None:
self.imet_id = imet_unique_id(_telemetry, custom=self.imet_location)
self.imet_id = imet_unique_id(_telemetry)
# Re-generate the datetime string.
_telemetry["datetime"] = _telemetry["datetime_dt"].strftime(
@ -1377,6 +1398,9 @@ class SondeDecoder(object):
if self.decoder is not None and (not nowait):
self.decoder.join()
if self.raw_file:
self.raw_file.close()
def running(self):
""" Check if the decoder subprocess is running.

2
auto_rx/autorx/sonde_specific.py
Wyświetl plik

@ -51,7 +51,7 @@ def fix_datetime(datetime_str, local_dt_str=None):
#
def imet_unique_id(telemetry, custom=""):
def imet_unique_id(telemetry, custom="SONDE"):
"""
Generate a 'unique' imet radiosonde ID based on the power-on time, frequency, and an optional location code.
This requires the following fields be present in the telemetry dict:

3
auto_rx/station.cfg.example
Wyświetl plik

@ -412,6 +412,9 @@ save_decode_audio = False
# Note: This will use a LOT of disk space.
save_decode_iq = False
# Save raw hexadecimal radiosonde frame data. This is useful to provide data for telemetry analysis.
# Raw hex data is saved to the logging directory with a filename of format YYYYMMDD-HHMMSS_<type>_<freq>.raw
save_raw_hex = False
#####################
# ADVANCED SETTINGS #

112
mk2a/mk2a_lms1680.c
Wyświetl plik

@ -596,82 +596,80 @@ void print_frame(int len) {
}
printf("\n");
}
else {
if (frame_bytes[OFS] == 0x4D && len/BITS > pos_FullID+4) {
if ( !crc_err ) {
if (frame_bytes[pos_SondeID] == frame_bytes[pos_FullID] &&
frame_bytes[pos_SondeID+1] == frame_bytes[pos_FullID+1]) {
ui32_t __id = (frame_bytes[pos_FullID+2]<<24) | (frame_bytes[pos_FullID+3]<<16)
| (frame_bytes[pos_FullID] << 8) | frame_bytes[pos_FullID+1];
gpx.id = __id;
}
if (frame_bytes[OFS] == 0x4D && len/BITS > pos_FullID+4) {
if ( !crc_err ) {
if (frame_bytes[pos_SondeID] == frame_bytes[pos_FullID] &&
frame_bytes[pos_SondeID+1] == frame_bytes[pos_FullID+1]) {
ui32_t __id = (frame_bytes[pos_FullID+2]<<24) | (frame_bytes[pos_FullID+3]<<16)
| (frame_bytes[pos_FullID] << 8) | frame_bytes[pos_FullID+1];
gpx.id = __id;
}
}
}
if (frame_bytes[OFS] == 0x54 && len/BITS > pos_GPSalt+4) {
if (frame_bytes[OFS] == 0x54 && len/BITS > pos_GPSalt+4) {
get_FrameNb();
get_GPStime();
get_GPSlat();
get_GPSlon();
get_GPSalt();
get_FrameNb();
get_GPStime();
get_GPSlat();
get_GPSlon();
get_GPSalt();
if ( !crc_err ) {
ui32_t _id = (frame_bytes[pos_SondeID]<<8) | frame_bytes[pos_SondeID+1];
if ((gpx.id & 0xFFFF) != _id) gpx.id = _id;
}
if (option_verbose && !crc_err) {
if (gpx.id & 0xFFFF0000) printf(" (%u)", gpx.id);
else if (gpx.id) printf(" (0x%04X)", gpx.id);
}
if ( !crc_err ) {
ui32_t _id = (frame_bytes[pos_SondeID]<<8) | frame_bytes[pos_SondeID+1];
if ((gpx.id & 0xFFFF) != _id) gpx.id = _id;
}
if (option_verbose && !crc_err) {
if (gpx.id & 0xFFFF0000) printf(" (%u)", gpx.id);
else if (gpx.id) printf(" (0x%04X)", gpx.id);
}
printf(" [%5d] ", gpx.frnr);
printf(" [%5d] ", gpx.frnr);
printf("%s ", weekday[gpx.wday]);
printf("%02d:%02d:%06.3f ", gpx.std, gpx.min, gpx.sek); // falls Rundung auf 60s: Ueberlauf
printf(" lat: %.5f ", gpx.lat);
printf(" lon: %.5f ", gpx.lon);
printf(" alt: %.2fm ", gpx.alt);
printf("%s ", weekday[gpx.wday]);
printf("%02d:%02d:%06.3f ", gpx.std, gpx.min, gpx.sek); // falls Rundung auf 60s: Ueberlauf
printf(" lat: %.5f ", gpx.lat);
printf(" lon: %.5f ", gpx.lon);
printf(" alt: %.2fm ", gpx.alt);
get_GPSvel24();
printf(" vH: %.1fm/s D: %.1f vV: %.1fm/s ", gpx.vH, gpx.vD, gpx.vV);
//if (option_verbose == 2) printf(" (%.1f ,%.1f,%.1f) ", gpx.vE, gpx.vN, gpx.vU);
get_GPSvel24();
printf(" vH: %.1fm/s D: %.1f vV: %.1fm/s ", gpx.vH, gpx.vD, gpx.vV);
//if (option_verbose == 2) printf(" (%.1f ,%.1f,%.1f) ", gpx.vE, gpx.vN, gpx.vU);
if (option_crc) {
if (crc_err==0) printf(" [OK]"); else printf(" [NO]");
}
if (option_crc) {
if (crc_err==0) printf(" [OK]"); else printf(" [NO]");
}
printf("\n");
printf("\n");
if (option_jsn) {
// Print JSON output required by auto_rx.
if (crc_err==0 && (gpx.id & 0xFFFF0000)) { // CRC-OK and FullID
if (gpx.prev_frnr != gpx.frnr) { //|| gpx.id != _id0
// UTC oder GPS?
char *ver_jsn = NULL;
printf("{ \"type\": \"%s\"", "LMS");
printf(", \"frame\": %d, \"id\": \"LMS6-%d\", \"datetime\": \"%02d:%02d:%06.3fZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f, \"vel_h\": %.5f, \"heading\": %.5f, \"vel_v\": %.5f",
gpx.frnr, gpx.id, gpx.std, gpx.min, gpx.sek, gpx.lat, gpx.lon, gpx.alt, gpx.vH, gpx.vD, gpx.vV );
printf(", \"subtype\": \"%s\"", "MK2A");
if (gpx.jsn_freq > 0) {
printf(", \"freq\": %d", gpx.jsn_freq);
}
#ifdef VER_JSN_STR
ver_jsn = VER_JSN_STR;
#endif
if (ver_jsn && *ver_jsn != '\0') printf(", \"version\": \"%s\"", ver_jsn);
printf(" }\n");
printf("\n");
gpx.prev_frnr = gpx.frnr;
if (option_jsn) {
// Print JSON output required by auto_rx.
if (crc_err==0 && (gpx.id & 0xFFFF0000)) { // CRC-OK and FullID
if (gpx.prev_frnr != gpx.frnr) { //|| gpx.id != _id0
// UTC oder GPS?
char *ver_jsn = NULL;
printf("{ \"type\": \"%s\"", "LMS");
printf(", \"frame\": %d, \"id\": \"LMS6-%d\", \"datetime\": \"%02d:%02d:%06.3fZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f, \"vel_h\": %.5f, \"heading\": %.5f, \"vel_v\": %.5f",
gpx.frnr, gpx.id, gpx.std, gpx.min, gpx.sek, gpx.lat, gpx.lon, gpx.alt, gpx.vH, gpx.vD, gpx.vV );
printf(", \"subtype\": \"%s\"", "MK2A");
if (gpx.jsn_freq > 0) {
printf(", \"freq\": %d", gpx.jsn_freq);
}
#ifdef VER_JSN_STR
ver_jsn = VER_JSN_STR;
#endif
if (ver_jsn && *ver_jsn != '\0') printf(", \"version\": \"%s\"", ver_jsn);
printf(" }\n");
printf("\n");
gpx.prev_frnr = gpx.frnr;
}
}
}
}
}