diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 1f61c66..03e7c5a 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -891,7 +891,10 @@ def main(): # Start our exporter options # Telemetry Logger if config["per_sonde_log"]: - _logger = TelemetryLogger(log_directory=logging_path) + _logger = TelemetryLogger( + log_directory=logging_path, + save_cal_data=config["save_cal_data"] + ) exporter_objects.append(_logger) exporter_functions.append(_logger.add) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index e7e21f9..2f98435 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ from queue import Queue # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.1-beta3" +__version__ = "1.7.1-beta4" # Global Variables diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index 27ffd17..07503a8 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -165,6 +165,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): "save_raw_hex": False, "save_system_log": False, "enable_debug_logging": False, + "save_cal_data": False, # URL for the Habitat DB Server. # As of July 2018 we send via sondehub.org, which will allow us to eventually transition away # from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists. @@ -775,6 +776,17 @@ def read_auto_rx_config(filename, no_sdr_test=False): ) auto_rx_config["wideband_sondes"] = False + # 1.7.1 - Save RS41 Calibration Data + try: + auto_rx_config["save_cal_data"] = config.getboolean( + "logging", "save_cal_data" + ) + except: + logging.warning( + "Config - Missing save_cal_data option (new in v1.7.1), using default (False)" + ) + auto_rx_config["save_cal_data"] = False + # If we are being called as part of a unit test, just return the config now. if no_sdr_test: return auto_rx_config diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 9f4d381..17bcd9b 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -246,6 +246,9 @@ class SondeDecoder(object): self.imet_prev_time = None self.imet_prev_frame = None + # Keep a record of which RS41 serials we have uploaded complete subframe data for. + self.rs41_subframe_uploads = [] + # This will become our decoder thread. self.decoder = None @@ -394,7 +397,7 @@ class SondeDecoder(object): if self.save_decode_audio: decode_cmd += f" tee {self.save_decode_audio_path} |" - decode_cmd += "./rs41mod --ptu2 --json 2>/dev/null" + decode_cmd += "./rs41mod --ptu2 --json --jsnsubfrm1 2>/dev/null" elif self.sonde_type == "RS92": # Decoding a RS92 requires either an ephemeris or an almanac file. @@ -830,7 +833,7 @@ class SondeDecoder(object): _baud_rate, ) - decode_cmd = f"./rs41mod --ptu2 --json --softin -i {self.raw_file_option} 2>/dev/null" + decode_cmd = f"./rs41mod --ptu2 --json --jsnsubfrm1 --softin -i {self.raw_file_option} 2>/dev/null" # RS41s transmit pulsed beacons - average over the last 2 frames, and use a peak-hold demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True) @@ -1705,6 +1708,19 @@ class SondeDecoder(object): "%Y-%m-%dT%H:%M:%SZ" ) + # RS41 Subframe Data Actions + # We only upload the subframe data once. + if 'rs41_calconf320' in _telemetry: + # Remove subframe data if we have already uploaded it once. + if _telemetry['id'] in self.rs41_subframe_uploads: + _telemetry.pop('rs41_calconf320') + else: + self.rs41_subframe_uploads.append(_telemetry['id']) + self.log_info(f"Received complete calibration dataset for {_telemetry['id']}.") + _telemetry['rs41_subframe'] = _telemetry['rs41_calconf320'] + _telemetry.pop('rs41_calconf320') + + # Grab a snapshot of modem statistics, if we are using an experimental decoder. if self.demod_stats is not None: diff --git a/auto_rx/autorx/logger.py b/auto_rx/autorx/logger.py index e00c40d..f893b7d 100644 --- a/auto_rx/autorx/logger.py +++ b/auto_rx/autorx/logger.py @@ -5,6 +5,7 @@ # Copyright (C) 2018 Mark Jessop # Released under GNU GPL v3 or later # +import codecs import datetime import glob import logging @@ -50,7 +51,9 @@ class TelemetryLogger(object): LOG_HEADER = "timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,pressure,type,freq_mhz,snr,f_error_hz,sats,batt_v,burst_timer,aux_data\n" - def __init__(self, log_directory="./log"): + def __init__(self, + log_directory="./log", + save_cal_data=False): """ Initialise and start a sonde logger. Args: @@ -59,6 +62,7 @@ class TelemetryLogger(object): """ self.log_directory = log_directory + self.save_cal_data = save_cal_data # Dictionary to contain file handles. # Each sonde id is added as a unique key. Under each key are the contents: @@ -214,6 +218,7 @@ class TelemetryLogger(object): self.open_logs[_id] = { "log": open(_log_file_name, "a"), "last_time": time.time(), + "subframe_saved": False } else: # Create a new log file. @@ -229,6 +234,7 @@ class TelemetryLogger(object): self.open_logs[_id] = { "log": open(_log_file_name, "a"), "last_time": time.time(), + "subframe_saved": False } # Write in a header line. @@ -244,6 +250,15 @@ class TelemetryLogger(object): self.open_logs[_id]["last_time"] = time.time() self.log_debug("Wrote line: %s" % _log_line.strip()) + # Save out RS41 subframe data once, if we have it. + if ('rs41_subframe' in telemetry) and self.save_cal_data: + if self.open_logs[_id]['subframe_saved'] == False: + self.open_logs[_id]['subframe_saved'] = self.write_rs41_subframe(telemetry) + + + + + def cleanup_logs(self): """ Close any open logs that have not had telemetry added in X seconds. """ @@ -262,6 +277,42 @@ class TelemetryLogger(object): except Exception as e: self.log_error("Error closing log for %s - %s" % (_id, str(e))) + def write_rs41_subframe(self, telemetry): + """ Write RS41 subframe data to disk """ + + _id = telemetry["id"] + _type = telemetry["type"] + + if 'aux' in telemetry: + _type += "-XDATA" + + _subframe_log_suffix = "%s_%s_%s_%d_subframe.bin" % ( + datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S"), + _id, + _type, + int(telemetry["freq_float"] * 1e3), # Convert frequency to kHz + ) + _log_file_name = os.path.join(self.log_directory, _subframe_log_suffix) + + + try: + _subframe_data = codecs.decode(telemetry['rs41_subframe'], 'hex') + except Exception as e: + self.log_error("Error parsing RS41 subframe data") + + if _subframe_data: + _subframe_file = open(_log_file_name, 'wb') + _subframe_file.write(_subframe_data) + _subframe_file.close() + + self.log_info(f"Wrote subframe data for {telemetry['id']} to {_subframe_log_suffix}") + return True + else: + return False + + + + def close(self): """ Close input processing thread. """ self.input_processing_running = False diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index 3d241b0..a917055 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -10,6 +10,8 @@ # Released under GNU GPL v3 or later # import autorx +import base64 +import codecs import datetime import glob import gzip @@ -297,6 +299,16 @@ class SondehubUploader(object): if "rs41_mainboard_fw" in telemetry: _output["rs41_mainboard_fw"] = str(telemetry["rs41_mainboard_fw"]) + if 'rs41_subframe' in telemetry: + # RS41 calibration subframe data. + # We try to base64 encode this. + try: + _calbytes = codecs.decode(telemetry['rs41_subframe'], 'hex') + _output['rs41_subframe'] = base64.b64encode(_calbytes).decode() + except Exception as e: + self.log_error(f"Error handling RS41 subframe data.") + + # Handle the additional SNR and frequency estimation if we have it if "snr" in telemetry: _output["snr"] = telemetry["snr"] diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index 18e5ae7..345c555 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -709,6 +709,14 @@ // Add data into the 'other' field. sonde_id_data.other = ""; + + if(sonde_id_data.hasOwnProperty('rs41_mainboard')){ + // Only print mainboard type if it's not the 'original' mainboard. + if(sonde_id_data.rs41_mainboard !== 'RSM412'){ + sonde_id_data.other += sonde_id_data.rs41_mainboard + " "; + } + } + // Burst timer for RS41s if (sonde_id_data.hasOwnProperty('bt')){ if ((sonde_id_data.bt >= 0) && (sonde_id_data.bt < 65535)) { diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index 7a8ce8e..edbde7f 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -458,6 +458,9 @@ save_system_log = False # auto_rx operational issues. enable_debug_logging = False +# Enable logging of RS41 Calibration data ('subframe' data) +# This is saved as a binary file with file suffix _subframe.bin +save_cal_data = False ########################### # WEB INTERFACE SETTINNGS # diff --git a/auto_rx/station.cfg.example.network b/auto_rx/station.cfg.example.network index f8ea1f2..4cd4618 100644 --- a/auto_rx/station.cfg.example.network +++ b/auto_rx/station.cfg.example.network @@ -458,6 +458,10 @@ save_system_log = False # auto_rx operational issues. enable_debug_logging = False +# Enable logging of RS41 Calibration data ('subframe' data) +# This is saved as a binary file with file suffix _subframe.bin +save_cal_data = False + ########################### # WEB INTERFACE SETTINNGS # ########################### diff --git a/demod/mod/rs41mod.c b/demod/mod/rs41mod.c index 0eb4bab..0beb630 100644 --- a/demod/mod/rs41mod.c +++ b/demod/mod/rs41mod.c @@ -65,6 +65,7 @@ typedef struct { i8_t aut; i8_t jsn; // JSON output (auto_rx) i8_t slt; // silent (only raw/json) + i8_t cal; // json cal/conf } option_t; typedef struct { @@ -117,6 +118,9 @@ typedef struct { ui8_t dfrm_bitscore[FRAME_LEN]; ui8_t calibytes[51*16]; ui8_t calfrchk[51]; + ui8_t calconf_complete; + ui8_t calconf_sent; + ui8_t *calconf_subfrm; // 1+16 byte cal/conf subframe float ptu_Rf1; // ref-resistor f1 (750 Ohm) float ptu_Rf2; // ref-resistor f2 (1100 Ohm) float ptu_co1[3]; // { -243.911 , 0.187654 , 8.2e-06 } @@ -267,15 +271,15 @@ float r4(ui8_t *bytes) { } */ -static int crc16(gpx_t *gpx, int start, int len) { +static int crc16(ui8_t data[], int len) { int crc16poly = 0x1021; int rem = 0xFFFF, i, j; int byte; - if (start+len+2 > FRAME_LEN) return -1; + //if (start+len+2 > FRAME_LEN) return -1; for (i = 0; i < len; i++) { - byte = gpx->frame[start+i]; + byte = data[i]; rem = rem ^ (byte << 8); for (j = 0; j < 8; j++) { if (rem & 0x8000) { @@ -298,7 +302,7 @@ static int check_CRC(gpx_t *gpx, ui32_t pos, ui32_t pck) { crclen = gpx->frame[pos+1]; if (pos + crclen + 4 > FRAME_LEN) return -1; crcdat = u2(gpx->frame+pos+2+crclen); - if ( crcdat != crc16(gpx, pos+2, crclen) ) { + if ( crcdat != crc16(gpx->frame+pos+2, crclen) ) { return 1; // CRC NO } else return 0; // CRC OK @@ -451,6 +455,8 @@ static int get_SondeID(gpx_t *gpx, int crc, int ofs) { // reset conf data memset(gpx->rstyp, 0, 9); memset(gpx->rsm, 0, 10); + gpx->calconf_complete = 0; + gpx->calconf_sent = 0; gpx->freq = 0; gpx->conf_fw = 0; gpx->conf_bt = 0; @@ -504,6 +510,19 @@ static int get_FrameConf(gpx_t *gpx, int ofs) { gpx->ecdat.last_calfrm = calfr; gpx->ecdat.last_calfrm_ts = gpx->ecdat.ts; + + if ( !gpx->calconf_complete ) { + int sum = 0; + for (i = 0; i < 51; i++) { // 0x00..0x32 + sum += gpx->calfrchk[i]; + } + if (sum == 51) { // count all subframes + int calconf_dat = gpx->calibytes[0] | (gpx->calibytes[1]<<8); + int calconf_crc = crc16(gpx->calibytes+2, 50*16-2); // subframe 0x32 not included (variable) + + if (calconf_dat == calconf_crc) gpx->calconf_complete = 1; + } + } } return err; @@ -1299,7 +1318,7 @@ static int get_Aux(gpx_t *gpx, int out, int pos) { auxlen = gpx->frame[pos+1]; auxcrc = gpx->frame[pos+2+auxlen] | (gpx->frame[pos+2+auxlen+1]<<8); - if ( auxcrc == crc16(gpx, pos+2, auxlen) ) { + if ( pos + auxlen + 4 <= FRAME_LEN && auxcrc == crc16(gpx->frame+pos+2, auxlen) ) { if (count7E == 0) { if (out) fprintf(stdout, "\n # xdata = "); } @@ -1372,6 +1391,8 @@ static int get_Calconf(gpx_t *gpx, int out, int ofs) { char rsmtyp[10]; int err = 0; + gpx->calconf_subfrm = gpx->frame+pos_CalData+ofs; + byte = gpx->frame[pos_CalData+ofs]; calfr = byte; err = check_CRC(gpx, pos_FRAME+ofs, pck_FRAME); @@ -2110,6 +2131,31 @@ static int print_position(gpx_t *gpx, int ec) { fprintf(stdout, ", \"rs41_mainboard_fw\": %d", gpx->conf_fw); } + if (gpx->option.cal == 1) { // cal/conf + if ( !gpx->calconf_sent && gpx->calconf_complete ) { + fprintf(stdout, ", \"rs41_calconf320\": \""); + for (int _j = 0; _j < 51*16; _j++) { + fprintf(stdout, "%02X", gpx->calibytes[_j]); + } + fprintf(stdout, "\""); + gpx->calconf_sent = 1; + } + if (gpx->calconf_subfrm[0] == 0x32) { + fprintf(stdout, ", \"rs41_conf0x32\": \""); + for (int _j = 0; _j < 16; _j++) { + fprintf(stdout, "%02X", gpx->calconf_subfrm[1+_j]); + } + fprintf(stdout, "\""); + } + } + if (gpx->option.cal == 2) { // cal/conf + fprintf(stdout, ", \"rs41_subfrm\": \"0x%02X:", gpx->calconf_subfrm[0]); + for (int _j = 0; _j < 16; _j++) { + fprintf(stdout, "%02X", gpx->calconf_subfrm[1+_j]); + } + fprintf(stdout, "\""); + } + // Include frequency derived from subframe information if available. if (gpx->freq > 0) { fprintf(stdout, ", \"tx_frequency\": %d", gpx->freq ); @@ -2444,6 +2490,8 @@ int main(int argc, char *argv[]) { if (frq < 300000000) frq = -1; cfreq = frq; } + else if (strcmp(*argv, "--jsnsubfrm1") == 0) { gpx.option.cal = 1; } // json cal/conf + else if (strcmp(*argv, "--jsnsubfrm2") == 0) { gpx.option.cal = 2; } // json cal/conf else if (strcmp(*argv, "--rawhex") == 0) { rawhex = 2; } // raw hex input else if (strcmp(*argv, "--xorhex") == 0) { rawhex = 2; xorhex = 1; } // raw xor input else if (strcmp(*argv, "-") == 0) { @@ -2495,6 +2543,13 @@ int main(int argc, char *argv[]) { // init gpx memcpy(gpx.frame, rs41_header_bytes, sizeof(rs41_header_bytes)); // 8 header bytes + gpx.calconf_subfrm = gpx.frame+pos_CalData; + if (gpx.option.cal) { + gpx.option.jsn = 1; + gpx.option.ecc = 2; + gpx.option.crc = 1; + } + if (cfreq > 0) gpx.jsn_freq = (cfreq+500)/1000;