Narrow allowed payload-system time window, add warnings for various issues.

2023-07-28 21:30:06 +09:30 · 2023-07-28 21:30:06 +09:30 · 948e97b3eb
commit 948e97b3eb
--- a/horusdemodlib/init.py
+++ b/horusdemodlib/init.py
@ -1 +1 @@
-__version__ = "0.3.11"
+__version__ = "0.3.12"
--- a/horusdemodlib/decoder.py
+++ b/horusdemodlib/decoder.py
@ -178,6 +178,11 @@ def decode_packet(data:bytes, packet_format:dict = None, ignore_crc:bool = False
            _ukhas_fields.append(_decoded_str)


+    # Check the payload ID if > 256 for a Horus v2 packet.
+    if _output['modulation'] == 'Horus Binary v2':
+        if _raw_fields[0] < 256:
+            logging.warning("Found Payload ID < 256 in a Horus Binary v2 packet! This may lead to undefined behaviour. Please use a payload ID > 256!")
+
    # Convert to a UKHAS-compliant string.
    _ukhas_str = ",".join(_ukhas_fields)
    _ukhas_crc = ukhas_crc(_ukhas_str.encode('ascii'))
--- a/horusdemodlib/sondehubamateur.py
+++ b/horusdemodlib/sondehubamateur.py
@ -155,6 +155,18 @@ class SondehubAmateurUploader(object):
        # Datetime
        try:
            _datetime = fix_datetime(telemetry['time'])
+
+            # Compare system time and payload time, to look for issues where system time is way out.
+            _timedelta = abs((_datetime - datetime.datetime.utcnow()).total_seconds())
+
+            if _timedelta > 3*60:
+                # Greater than 3 minutes time difference. Discard packet in this case.
+                self.log_error("Payload and Receiver times are offset by more than 3 minutes. Either payload does not have GNSS lock, or your system time is not set correctly. Not uploading.")
+                return None
+
+            if _timedelta > 60:
+                self.log_warning("Payload and Receiver times are offset by more than 1 minute. Either payload does not have GNSS lock, or your system time is not set correctly.")
+
            _output["datetime"] = _datetime.strftime(
                "%Y-%m-%dT%H:%M:%S.%fZ"
            )
@ -165,6 +177,8 @@ class SondehubAmateurUploader(object):
            self.log_debug("Offending datetime_dt: %s" % str(telemetry["time"]))
            return None

+
+
        # Callsign - Break if this is an unknown payload ID.
        if telemetry["callsign"] == "UNKNOWN_PAYLOAD_ID":
            self.log_error("Not uploading telemetry from unknown payload ID. Is your payload ID list old?")
@ -183,10 +197,6 @@ class SondehubAmateurUploader(object):
        _output["lon"] = telemetry["longitude"]
        _output["alt"] = telemetry["altitude"]

-        if (_output["lat"] == 0.0) and (_output["lon"] == 0.0):
-            self.log_error("Lat/Lon both 0.0 - not uploading telemetry.")
-            return None
-
        # # Optional Fields
        if "temperature" in telemetry:
            if telemetry["temperature"] > -273.15:
@ -195,10 +205,6 @@ class SondehubAmateurUploader(object):
        if "satellites" in telemetry:
            _output["sats"] = telemetry["satellites"]

-            if _output["sats"] == 0:
-                self.log_error("Satellites field provided, and is 0. Not uploading due to potentially inaccurate position.")
-                return None
-
        if "battery_voltage" in telemetry:
            if telemetry["battery_voltage"] >= 0.0:
                _output["batt"] = telemetry["battery_voltage"]
@ -338,7 +344,7 @@ class SondehubAmateurUploader(object):
                _upload_success = True
                break

-            elif _req.status_code == 500:
+            elif _req.status_code in [500,501,502,503,504]:
                # Server Error, Retry.
                _retries += 1
                continue
--- a/horusdemodlib/uploader.py
+++ b/horusdemodlib/uploader.py
@ -98,6 +98,9 @@ def main():
    else:
        _logfile = None

+    # Some variables to handle re-downloading of payload ID lists.
+    min_download_time = 30*60 # Only try and download new payload ID / custom field lists every 30 min.
+    next_download_time = time.time()

    if args.rtty == False:

@ -106,7 +109,7 @@ def main():
            horusdemodlib.payloads.HORUS_PAYLOAD_LIST = read_payload_list(filename=args.payload_list)
            horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = read_custom_field_list(filename=args.custom_fields)
        else:
-            # Downlaod
+            # Download
            horusdemodlib.payloads.HORUS_PAYLOAD_LIST = init_payload_id_list(filename=args.payload_list)
            horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = init_custom_field_list(filename=args.custom_fields)
            
@ -121,16 +124,6 @@ def main():
    else:
        _listener_freq_str = ""

-    # Habitat uploader disabled as of 2022-12-18
-    # habitat_uploader = HabitatUploader(
-    #     user_callsign = user_config['user_call'],
-    #     listener_lat = user_config['station_lat'],
-    #     listener_lon = user_config['station_lon'],
-    #     listener_radio = user_config['radio_comment'] + _listener_freq_str,
-    #     listener_antenna = user_config['antenna_comment'],
-    #     inhibit=args.noupload
-    # )
-
    if user_config['station_lat'] == 0.0 and user_config['station_lon'] == 0.0:
        _sondehub_user_pos = None
    else:
@ -140,7 +133,7 @@ def main():
        upload_rate = 2,
        user_callsign = user_config['user_call'],
        user_position = _sondehub_user_pos,
-        user_radio = user_config['radio_comment'],
+        user_radio = user_config['radio_comment'] + _listener_freq_str,
        user_antenna = user_config['antenna_comment'],
        software_name = "horusdemodlib",
        software_version = horusdemodlib.__version__,
@ -225,6 +218,23 @@ def main():
                try:
                    _decoded = decode_packet(_binary_string)
                    # If we get here, we have a valid packet!
+
+                    if (_decoded['callsign'] == "UNKNOWN_PAYLOAD_ID") and not args.nodownload:
+                        # We haven't seen this payload ID. Our payload ID list might be out of date.
+                        if time.time() > next_download_time:
+                            logging.info("Observed unknown Payload ID, attempting to re-download lists.")
+                            
+                            # Download lists.
+                            horusdemodlib.payloads.HORUS_PAYLOAD_LIST = init_payload_id_list(filename=args.payload_list)
+                            horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = init_custom_field_list(filename=args.custom_fields)
+                            
+                            # Update next_download_time so we don't re-attempt to download with every new packet.
+                            next_download_time = time.time() + min_download_time
+
+                            # Re-attempt to decode the packet.
+                            _decoded = decode_packet(_binary_string)
+                            if _decoded['callsign'] != "UNKNOWN_PAYLOAD_ID":
+                                logging.info(f"Payload found in new payload ID list - {_decoded['callsign']}")
                    
                    # Add in SNR data.
                    _snr = demod_stats.snr
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,6 +1,6 @@
 [tool.poetry]
 name = "horusdemodlib"
-version = "0.3.11"
+version = "0.3.12"
 description = "Project Horus HAB Telemetry Demodulators"
 authors = ["Mark Jessop"]
 license = "LGPL-2.1-or-later"
--- a/scripts/docker_dual_4fsk.sh
+++ b/scripts/docker_dual_4fsk.sh
@ -47,4 +47,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($MFSK1_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
--- a/scripts/docker_dual_rtty_4fsk.sh
+++ b/scripts/docker_dual_rtty_4fsk.sh
@ -49,4 +49,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($RTTY_CENTRE / $MFSK_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python3 -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python3 -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
--- a/scripts/docker_single.sh
+++ b/scripts/docker_single.sh
@ -34,4 +34,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($SDR_RX_FREQ) and 'target' signal frequency ($RXFREQ)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
--- a/start_dual_4fsk.sh
+++ b/start_dual_4fsk.sh
@ -29,6 +29,11 @@ MFSK2_SIGNAL=15000
 # but the higher the chance that the modem will lock on to a strong spurious signal.
 RXBANDWIDTH=5000

+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@ -113,4 +118,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($MFSK1_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
--- a/start_dual_rtty_4fsk.sh
+++ b/start_dual_rtty_4fsk.sh
@ -31,6 +31,11 @@ MFSK_SIGNAL=15000
 # but the higher the chance that the modem will lock on to a strong spurious signal.
 RXBANDWIDTH=8000

+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@ -115,4 +120,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($RTTY_CENTRE / $MFSK_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
--- a/start_rtlsdr.sh
+++ b/start_rtlsdr.sh
@ -15,6 +15,12 @@ cd /home/pi/horusdemodlib/
 # Note: The SDR will be tuned to RXBANDWIDTH/2 below this frequency.
 RXFREQ=434200000

+
+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@ -91,4 +97,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($SDR_RX_FREQ) and 'target' signal frequency ($RXFREQ)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
--- a/user.env.example
+++ b/user.env.example
@ -1,5 +1,10 @@
 ### General SDR settings ###

+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.