diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 1f61c66..03e7c5a 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -891,7 +891,10 @@ def main(): # Start our exporter options # Telemetry Logger if config["per_sonde_log"]: - _logger = TelemetryLogger(log_directory=logging_path) + _logger = TelemetryLogger( + log_directory=logging_path, + save_cal_data=config["save_cal_data"] + ) exporter_objects.append(_logger) exporter_functions.append(_logger.add) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index d4a6269..427e6c1 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ from queue import Queue # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.0" +__version__ = "1.7.1" # Global Variables diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index 27ffd17..07503a8 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -165,6 +165,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): "save_raw_hex": False, "save_system_log": False, "enable_debug_logging": False, + "save_cal_data": False, # URL for the Habitat DB Server. # As of July 2018 we send via sondehub.org, which will allow us to eventually transition away # from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists. @@ -775,6 +776,17 @@ def read_auto_rx_config(filename, no_sdr_test=False): ) auto_rx_config["wideband_sondes"] = False + # 1.7.1 - Save RS41 Calibration Data + try: + auto_rx_config["save_cal_data"] = config.getboolean( + "logging", "save_cal_data" + ) + except: + logging.warning( + "Config - Missing save_cal_data option (new in v1.7.1), using default (False)" + ) + auto_rx_config["save_cal_data"] = False + # If we are being called as part of a unit test, just return the config now. if no_sdr_test: return auto_rx_config diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 58c6792..75679a2 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -246,6 +246,9 @@ class SondeDecoder(object): self.imet_prev_time = None self.imet_prev_frame = None + # Keep a record of which RS41 serials we have uploaded complete subframe data for. + self.rs41_subframe_uploads = [] + # This will become our decoder thread. self.decoder = None @@ -394,7 +397,7 @@ class SondeDecoder(object): if self.save_decode_audio: decode_cmd += f" tee {self.save_decode_audio_path} |" - decode_cmd += "./rs41mod --ptu2 --json 2>/dev/null" + decode_cmd += "./rs41mod --ptu2 --json --jsnsubfrm1 2>/dev/null" elif self.sonde_type == "RS92": # Decoding a RS92 requires either an ephemeris or an almanac file. @@ -830,7 +833,7 @@ class SondeDecoder(object): _baud_rate, ) - decode_cmd = f"./rs41mod --ptu2 --json --softin -i {self.raw_file_option} 2>/dev/null" + decode_cmd = f"./rs41mod --ptu2 --json --jsnsubfrm1 --softin -i {self.raw_file_option} 2>/dev/null" # RS41s transmit pulsed beacons - average over the last 2 frames, and use a peak-hold demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True) @@ -1602,8 +1605,9 @@ class SondeDecoder(object): # which is most likely an Ozone sensor (though could be something different!) # We append -Ozone to the sonde type field to indicate this. # TODO: Decode device ID from aux field to indicate what the aux payload actually is? - if "aux" in _telemetry: - _telemetry["type"] += "-Ozone" + # 2023-10 - disabled this addition. Can be too misleading. -XDATA now appended on the web interface only. + # if "aux" in _telemetry: + # _telemetry["type"] += "-Ozone" # iMet Specific actions if self.sonde_type == "IMET": @@ -1704,6 +1708,19 @@ class SondeDecoder(object): "%Y-%m-%dT%H:%M:%SZ" ) + # RS41 Subframe Data Actions + # We only upload the subframe data once. + if 'rs41_calconf51x16' in _telemetry: + # Remove subframe data if we have already uploaded it once. + if _telemetry['id'] in self.rs41_subframe_uploads: + _telemetry.pop('rs41_calconf51x16') + else: + self.rs41_subframe_uploads.append(_telemetry['id']) + self.log_info(f"Received complete calibration dataset for {_telemetry['id']}.") + _telemetry['rs41_subframe'] = _telemetry['rs41_calconf51x16'] + _telemetry.pop('rs41_calconf51x16') + + # Grab a snapshot of modem statistics, if we are using an experimental decoder. if self.demod_stats is not None: diff --git a/auto_rx/autorx/logger.py b/auto_rx/autorx/logger.py index 1b0fa29..f893b7d 100644 --- a/auto_rx/autorx/logger.py +++ b/auto_rx/autorx/logger.py @@ -5,6 +5,7 @@ # Copyright (C) 2018 Mark Jessop # Released under GNU GPL v3 or later # +import codecs import datetime import glob import logging @@ -50,7 +51,9 @@ class TelemetryLogger(object): LOG_HEADER = "timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,pressure,type,freq_mhz,snr,f_error_hz,sats,batt_v,burst_timer,aux_data\n" - def __init__(self, log_directory="./log"): + def __init__(self, + log_directory="./log", + save_cal_data=False): """ Initialise and start a sonde logger. Args: @@ -59,6 +62,7 @@ class TelemetryLogger(object): """ self.log_directory = log_directory + self.save_cal_data = save_cal_data # Dictionary to contain file handles. # Each sonde id is added as a unique key. Under each key are the contents: @@ -199,6 +203,9 @@ class TelemetryLogger(object): _id = telemetry["id"] _type = telemetry["type"] + if 'aux' in telemetry: + _type += "-XDATA" + # If there is no log open for the current ID check to see if there is an existing (closed) log file, and open it. if _id not in self.open_logs: _search_string = os.path.join(self.log_directory, "*%s_*_sonde.log" % (_id)) @@ -211,6 +218,7 @@ class TelemetryLogger(object): self.open_logs[_id] = { "log": open(_log_file_name, "a"), "last_time": time.time(), + "subframe_saved": False } else: # Create a new log file. @@ -226,6 +234,7 @@ class TelemetryLogger(object): self.open_logs[_id] = { "log": open(_log_file_name, "a"), "last_time": time.time(), + "subframe_saved": False } # Write in a header line. @@ -241,6 +250,15 @@ class TelemetryLogger(object): self.open_logs[_id]["last_time"] = time.time() self.log_debug("Wrote line: %s" % _log_line.strip()) + # Save out RS41 subframe data once, if we have it. + if ('rs41_subframe' in telemetry) and self.save_cal_data: + if self.open_logs[_id]['subframe_saved'] == False: + self.open_logs[_id]['subframe_saved'] = self.write_rs41_subframe(telemetry) + + + + + def cleanup_logs(self): """ Close any open logs that have not had telemetry added in X seconds. """ @@ -259,6 +277,42 @@ class TelemetryLogger(object): except Exception as e: self.log_error("Error closing log for %s - %s" % (_id, str(e))) + def write_rs41_subframe(self, telemetry): + """ Write RS41 subframe data to disk """ + + _id = telemetry["id"] + _type = telemetry["type"] + + if 'aux' in telemetry: + _type += "-XDATA" + + _subframe_log_suffix = "%s_%s_%s_%d_subframe.bin" % ( + datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S"), + _id, + _type, + int(telemetry["freq_float"] * 1e3), # Convert frequency to kHz + ) + _log_file_name = os.path.join(self.log_directory, _subframe_log_suffix) + + + try: + _subframe_data = codecs.decode(telemetry['rs41_subframe'], 'hex') + except Exception as e: + self.log_error("Error parsing RS41 subframe data") + + if _subframe_data: + _subframe_file = open(_log_file_name, 'wb') + _subframe_file.write(_subframe_data) + _subframe_file.close() + + self.log_info(f"Wrote subframe data for {telemetry['id']} to {_subframe_log_suffix}") + return True + else: + return False + + + + def close(self): """ Close input processing thread. """ self.input_processing_running = False diff --git a/auto_rx/autorx/scan.py b/auto_rx/autorx/scan.py index 45f84d4..cc2d791 100644 --- a/auto_rx/autorx/scan.py +++ b/auto_rx/autorx/scan.py @@ -533,11 +533,13 @@ def detect_sonde( ) _sonde_type = "IMET" elif "IMET1" in _type: + # This could actually be a wideband iMet sonde. We treat this as a IMET4. logging.debug( - "Scanner (%s) - Detected a iMet Sonde! (Type %s - Unsupported) (Score: %.2f)" + "Scanner (%s) - Possible detection of a Wideband iMet Sonde! (Type %s) (Score: %.2f)" % (_sdr_name, _type, _score) ) - _sonde_type = "IMET1" + # Override the type to IMET4. + _sonde_type = "IMET" elif "IMETafsk" in _type: logging.debug( "Scanner (%s) - Detected a iMet Sonde! (Type %s - Unsupported) (Score: %.2f)" diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index fbb3b03..a917055 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -10,6 +10,8 @@ # Released under GNU GPL v3 or later # import autorx +import base64 +import codecs import datetime import glob import gzip @@ -291,6 +293,22 @@ class SondehubUploader(object): if "ref_datetime" in telemetry: _output["ref_datetime"] = telemetry["ref_datetime"] + if "rs41_mainboard" in telemetry: + _output["rs41_mainboard"] = telemetry["rs41_mainboard"] + + if "rs41_mainboard_fw" in telemetry: + _output["rs41_mainboard_fw"] = str(telemetry["rs41_mainboard_fw"]) + + if 'rs41_subframe' in telemetry: + # RS41 calibration subframe data. + # We try to base64 encode this. + try: + _calbytes = codecs.decode(telemetry['rs41_subframe'], 'hex') + _output['rs41_subframe'] = base64.b64encode(_calbytes).decode() + except Exception as e: + self.log_error(f"Error handling RS41 subframe data.") + + # Handle the additional SNR and frequency estimation if we have it if "snr" in telemetry: _output["snr"] = telemetry["snr"] diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index 6efba9a..345c555 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -709,6 +709,14 @@ // Add data into the 'other' field. sonde_id_data.other = ""; + + if(sonde_id_data.hasOwnProperty('rs41_mainboard')){ + // Only print mainboard type if it's not the 'original' mainboard. + if(sonde_id_data.rs41_mainboard !== 'RSM412'){ + sonde_id_data.other += sonde_id_data.rs41_mainboard + " "; + } + } + // Burst timer for RS41s if (sonde_id_data.hasOwnProperty('bt')){ if ((sonde_id_data.bt >= 0) && (sonde_id_data.bt < 65535)) { @@ -719,6 +727,11 @@ sonde_id_data.other += sonde_id_data.batt.toFixed(1) + " V"; } + if (sonde_id_data.hasOwnProperty('aux')){ + sonde_id_data.type += "-XDATA"; + } + + telem_data.push(sonde_id_data); }); } diff --git a/auto_rx/autorx/utils.py b/auto_rx/autorx/utils.py index f9a7cc3..9c789f0 100644 --- a/auto_rx/autorx/utils.py +++ b/auto_rx/autorx/utils.py @@ -196,6 +196,8 @@ def short_type_lookup(type_name): return "Lockheed Martin LMS6-1680" elif type_name == "IMET": return "Intermet Systems iMet-1/4" + elif type_name == "IMET-XDATA": + return "Intermet Systems iMet-1/4 + XDATA" elif type_name == "IMET5": return "Intermet Systems iMet-5x" elif type_name == "MEISEI": @@ -238,6 +240,8 @@ def short_short_type_lookup(type_name): return "LMS6-1680" elif type_name == "IMET": return "iMet-1/4" + elif type_name == "IMET-XDATA": + return "iMet-1/4" elif type_name == "IMET5": return "iMet-5x" elif type_name == "MEISEI": diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index 7a8ce8e..edbde7f 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -458,6 +458,9 @@ save_system_log = False # auto_rx operational issues. enable_debug_logging = False +# Enable logging of RS41 Calibration data ('subframe' data) +# This is saved as a binary file with file suffix _subframe.bin +save_cal_data = False ########################### # WEB INTERFACE SETTINNGS # diff --git a/auto_rx/station.cfg.example.network b/auto_rx/station.cfg.example.network index f8ea1f2..4cd4618 100644 --- a/auto_rx/station.cfg.example.network +++ b/auto_rx/station.cfg.example.network @@ -458,6 +458,10 @@ save_system_log = False # auto_rx operational issues. enable_debug_logging = False +# Enable logging of RS41 Calibration data ('subframe' data) +# This is saved as a binary file with file suffix _subframe.bin +save_cal_data = False + ########################### # WEB INTERFACE SETTINNGS # ########################### diff --git a/demod/mod/rs41mod.c b/demod/mod/rs41mod.c index 38a5cc3..937f7ed 100644 --- a/demod/mod/rs41mod.c +++ b/demod/mod/rs41mod.c @@ -65,6 +65,7 @@ typedef struct { i8_t aut; i8_t jsn; // JSON output (auto_rx) i8_t slt; // silent (only raw/json) + i8_t cal; // json cal/conf } option_t; typedef struct { @@ -117,6 +118,9 @@ typedef struct { ui8_t dfrm_bitscore[FRAME_LEN]; ui8_t calibytes[51*16]; ui8_t calfrchk[51]; + ui8_t calconf_complete; + ui8_t calconf_sent; + ui8_t *calconf_subfrm; // 1+16 byte cal/conf subframe float ptu_Rf1; // ref-resistor f1 (750 Ohm) float ptu_Rf2; // ref-resistor f2 (1100 Ohm) float ptu_co1[3]; // { -243.911 , 0.187654 , 8.2e-06 } @@ -139,6 +143,7 @@ typedef struct { ui16_t conf_cd; // kill countdown (sec) (kt or bt) ui8_t conf_bk; // burst kill char rstyp[9]; // RS41-SG, RS41-SGP + char rsm[10]; // RSM421 int aux; char xdata[XDATA_LEN+16]; // xdata: aux_str1#aux_str2 ... option_t option; @@ -266,15 +271,15 @@ float r4(ui8_t *bytes) { } */ -static int crc16(gpx_t *gpx, int start, int len) { +static int crc16(ui8_t data[], int len) { int crc16poly = 0x1021; int rem = 0xFFFF, i, j; int byte; - if (start+len+2 > FRAME_LEN) return -1; + //if (start+len+2 > FRAME_LEN) return -1; for (i = 0; i < len; i++) { - byte = gpx->frame[start+i]; + byte = data[i]; rem = rem ^ (byte << 8); for (j = 0; j < 8; j++) { if (rem & 0x8000) { @@ -297,7 +302,7 @@ static int check_CRC(gpx_t *gpx, ui32_t pos, ui32_t pck) { crclen = gpx->frame[pos+1]; if (pos + crclen + 4 > FRAME_LEN) return -1; crcdat = u2(gpx->frame+pos+2+crclen); - if ( crcdat != crc16(gpx, pos+2, crclen) ) { + if ( crcdat != crc16(gpx->frame+pos+2, crclen) ) { return 1; // CRC NO } else return 0; // CRC OK @@ -330,7 +335,8 @@ GPS chip: ublox UBX-G6010-ST #define pos_Calburst 0x05E // 1 byte, calfr 0x02 // ? #define pos_Caltimer 0x05A // 2 byte, calfr 0x02 ? #define pos_CalRSTyp 0x05B // 8 byte, calfr 0x21 (+2 byte in 0x22?) - // weitere chars in calfr 0x22/0x23; weitere ID + // weitere chars in calfr 0x22/0x23; weitere ID (RSM) +#define pos_CalRSM 0x055 // 6 byte, calfr 0x22 #define crc_PTU (1<<1) #define xor_PTU 0xE388 // ^0x99A2=0x0x7A2A @@ -448,6 +454,9 @@ static int get_SondeID(gpx_t *gpx, int crc, int ofs) { memset(gpx->calfrchk, 0, 51); // 0x00..0x32 // reset conf data memset(gpx->rstyp, 0, 9); + memset(gpx->rsm, 0, 10); + gpx->calconf_complete = 0; + gpx->calconf_sent = 0; gpx->freq = 0; gpx->conf_fw = 0; gpx->conf_bt = 0; @@ -501,6 +510,19 @@ static int get_FrameConf(gpx_t *gpx, int ofs) { gpx->ecdat.last_calfrm = calfr; gpx->ecdat.last_calfrm_ts = gpx->ecdat.ts; + + if ( !gpx->calconf_complete ) { + int sum = 0; + for (i = 0; i < 51; i++) { // 0x00..0x32 + sum += gpx->calfrchk[i]; + } + if (sum == 51) { // count all subframes + int calconf_dat = gpx->calibytes[0] | (gpx->calibytes[1]<<8); + int calconf_crc = crc16(gpx->calibytes+2, 50*16-2); // subframe 0x32 not included (variable) + + if (calconf_dat == calconf_crc) gpx->calconf_complete = 1; + } + } } return err; @@ -1296,7 +1318,7 @@ static int get_Aux(gpx_t *gpx, int out, int pos) { auxlen = gpx->frame[pos+1]; auxcrc = gpx->frame[pos+2+auxlen] | (gpx->frame[pos+2+auxlen+1]<<8); - if ( auxcrc == crc16(gpx, pos+2, auxlen) ) { + if ( pos + auxlen + 4 <= FRAME_LEN && auxcrc == crc16(gpx->frame+pos+2, auxlen) ) { if (count7E == 0) { if (out) fprintf(stdout, "\n # xdata = "); } @@ -1366,8 +1388,11 @@ static int get_Calconf(gpx_t *gpx, int out, int ofs) { ui16_t fw = 0; int freq = 0, f0 = 0, f1 = 0; char sondetyp[9]; + char rsmtyp[10]; int err = 0; + gpx->calconf_subfrm = gpx->frame+pos_CalData+ofs; + byte = gpx->frame[pos_CalData+ofs]; calfr = byte; err = check_CRC(gpx, pos_FRAME+ofs, pck_FRAME); @@ -1451,6 +1476,17 @@ static int get_Calconf(gpx_t *gpx, int out, int ofs) { } } } + + if (calfr == 0x22) { + for (i = 0; i < 10; i++) rsmtyp[i] = 0; + for (i = 0; i < 8; i++) { + byte = gpx->frame[pos_CalRSM+ofs + i]; + if ((byte >= 0x20) && (byte < 0x7F)) rsmtyp[i] = byte; + else /*if (byte == 0x00)*/ rsmtyp[i] = '\0'; + } + if (out && gpx->option.vbs) fprintf(stdout, ": %s ", rsmtyp); + strcpy(gpx->rsm, rsmtyp); + } } return 0; @@ -2088,6 +2124,45 @@ static int print_position(gpx_t *gpx, int ec) { if (gpx->freq > 0) fq_kHz = gpx->freq; fprintf(stdout, ", \"freq\": %d", fq_kHz); } + if (*gpx->rsm) { // RSM type + fprintf(stdout, ", \"rs41_mainboard\": \"%s\"", gpx->rsm); + } + if (gpx->conf_fw) { // firmware + fprintf(stdout, ", \"rs41_mainboard_fw\": %d", gpx->conf_fw); + } + + if (gpx->option.cal == 1) { // cal/conf + int _j; + if ( !gpx->calconf_sent && gpx->calconf_complete ) { + /* + fprintf(stdout, ", \"rs41_calconf320h\": \""); // only constant/crc part + for (int _j = 0; _j < 50*16; _j++) { + fprintf(stdout, "%02X", gpx->calibytes[_j]); + } + */ + fprintf(stdout, ", \"rs41_calconf51x16\": \""); + for (_j = 0; _j < 51*16; _j++) { + fprintf(stdout, "%02X", gpx->calibytes[_j]); + } + fprintf(stdout, "\""); + gpx->calconf_sent = 1; + } + if (gpx->calconf_subfrm[0] == 0x32) { + fprintf(stdout, ", \"rs41_conf0x32\": \""); + for (_j = 0; _j < 16; _j++) { + fprintf(stdout, "%02X", gpx->calconf_subfrm[1+_j]); + } + fprintf(stdout, "\""); + } + } + if (gpx->option.cal == 2) { // cal/conf + int _j; + fprintf(stdout, ", \"rs41_subfrm\": \"0x%02X:", gpx->calconf_subfrm[0]); + for (_j = 0; _j < 16; _j++) { + fprintf(stdout, "%02X", gpx->calconf_subfrm[1+_j]); + } + fprintf(stdout, "\""); + } // Include frequency derived from subframe information if available. if (gpx->freq > 0) { @@ -2423,6 +2498,8 @@ int main(int argc, char *argv[]) { if (frq < 300000000) frq = -1; cfreq = frq; } + else if (strcmp(*argv, "--jsnsubfrm1") == 0) { gpx.option.cal = 1; } // json cal/conf + else if (strcmp(*argv, "--jsnsubfrm2") == 0) { gpx.option.cal = 2; } // json cal/conf else if (strcmp(*argv, "--rawhex") == 0) { rawhex = 2; } // raw hex input else if (strcmp(*argv, "--xorhex") == 0) { rawhex = 2; xorhex = 1; } // raw xor input else if (strcmp(*argv, "-") == 0) { @@ -2474,6 +2551,13 @@ int main(int argc, char *argv[]) { // init gpx memcpy(gpx.frame, rs41_header_bytes, sizeof(rs41_header_bytes)); // 8 header bytes + gpx.calconf_subfrm = gpx.frame+pos_CalData; + if (gpx.option.cal) { + gpx.option.jsn = 1; + gpx.option.ecc = 2; + gpx.option.crc = 1; + } + if (cfreq > 0) gpx.jsn_freq = (cfreq+500)/1000;