Added: usb_reset ability, updates to use new rtlsdr bias tee -T options, oziplotter push, hopefully better frequency scanning.

pull/13/head
Mark Jessop 2017-07-16 19:41:13 +09:30
rodzic a350d1ad5b
commit c217f39f36
6 zmienionych plików z 196 dodań i 43 usunięć

Wyświetl plik

@ -15,6 +15,7 @@ Features:
* Uploading to:
* APRS, with user-definable position comment.
* Habitat
* OziPlotter (Project Horus Offline Mapping)
Dependencies
------------
@ -23,8 +24,8 @@ Dependencies
* numpy
* crcmod
* Also needs (grab from apt-get):
* rtl-sdr
* sox
* If you wish to use an RTLSDR with a bias tee, you will need a version of rtl-sdr which is newer than 2017-06-11. This may me having to grab the latest git version from: https://github.com/osmocom/rtl-sdr
Usage
-----

Wyświetl plik

@ -22,8 +22,6 @@
#
# TODO:
# [ ] Fix user gain setting issues. (gain='automatic' = no decode?!)
# [ ] Better peak signal detection. (Maybe convolve known spectral masks over power data?)
# [ ] Figure out better quantization settings.
# [ ] Use FSK demod from codec2-dev ?
# [ ] Storage of flight information in some kind of database.
# [ ] Local frequency blacklist, to speed up scan times.
@ -41,6 +39,7 @@ import subprocess
import traceback
from aprs_utils import *
from habitat_utils import *
from ozi_utils import *
from threading import Thread
from StringIO import StringIO
from findpeaks import *
@ -56,6 +55,10 @@ HABITAT_OUTPUT_ENABLED = False
INTERNET_PUSH_RUNNING = True
internet_push_queue = Queue.Queue()
# Second Queue for OziPlotter outputs, since we want this to run at a faster rate.
OZI_PUSH_RUNNING = True
ozi_push_queue = Queue.Queue()
# Flight Statistics data
# stores copies of the telemetry dictionary returned by process_rs_line.
@ -66,16 +69,20 @@ flight_stats = {
}
def run_rtl_power(start, stop, step, filename="log_power.csv", dwell = 20, ppm = 0, gain = 'automatic'):
def run_rtl_power(start, stop, step, filename="log_power.csv", dwell = 20, ppm = 0, gain = 'automatic', bias = False):
""" Run rtl_power, with a timeout"""
# rtl_power -f 400400000:403500000:800 -i20 -1 log_power.csv
rtl_power_cmd = "timeout %d rtl_power -f %d:%d:%d -i %d -1 -p %d %s" % (dwell+10, start, stop, step, dwell, int(ppm), filename)
# Add a -T option if bias is enabled
bias_option = "-T " if bias else ""
# Added -k 5 option, to SIGKILL rtl_power 5 seconds after the regular timeout expires.
rtl_power_cmd = "timeout -k 5 %d rtl_power %s-f %d:%d:%d -i %d -1 -c 20%% -p %d %s" % (dwell+10, bias_option, start, stop, step, dwell, int(ppm), filename)
logging.info("Running frequency scan.")
ret_code = os.system(rtl_power_cmd)
if ret_code == 1:
logging.critical("rtl_power call failed!")
sys.exit(1)
return False
else:
return True
@ -110,14 +117,19 @@ def read_rtl_power(filename):
freq_step = float(fields[4])
n_samples = int(fields[5])
freq_range = np.arange(start_freq,stop_freq,freq_step)
#freq_range = np.arange(start_freq,stop_freq,freq_step)
samples = np.loadtxt(StringIO(",".join(fields[6:])),delimiter=',')
freq_range = np.linspace(start_freq,stop_freq,len(samples))
# Add frequency range and samples to output buffers.
freq = np.append(freq, freq_range)
power = np.append(power, samples)
f.close()
# Sanitize power values, to remove the nan's that rtl_power puts in there occasionally.
power = np.nan_to_num(power)
return (freq, power, freq_step)
@ -128,16 +140,15 @@ def quantize_freq(freq_list, quantize=5000):
def detect_sonde(frequency, ppm=0, gain='automatic', bias=False):
""" Receive some FM and attempt to detect the presence of a radiosonde. """
rx_test_command = "timeout 10s rtl_fm -p %d -M fm -s 15k -f %d 2>/dev/null |" % (int(ppm), frequency)
# Add a -T option if bias is enabled
bias_option = "-T " if bias else ""
rx_test_command = "timeout 10s rtl_fm %s-p %d -M fm -s 15k -f %d 2>/dev/null |" % (bias_option, int(ppm), frequency)
rx_test_command += "sox -t raw -r 15k -e s -b 16 -c 1 - -r 48000 -t wav - highpass 20 2>/dev/null |"
rx_test_command += "./rs_detect -z -t 8 2>/dev/null"
logging.info("Attempting sonde detection on %.3f MHz" % (frequency/1e6))
# Enable Bias-Tee if required.
if bias:
os.system('rtl_biast -b 1')
ret_code = os.system(rx_test_command)
ret_code = ret_code >> 8
@ -151,6 +162,30 @@ def detect_sonde(frequency, ppm=0, gain='automatic', bias=False):
else:
return None
def reset_rtlsdr():
""" Attempt to perform a USB Reset on all attached RTLSDRs. This uses the usb_reset binary from ../scan"""
lsusb_output = subprocess.check_output(['lsusb'])
try:
devices = lsusb_output.split('\n')
for device in devices:
if 'RTL2838' in device:
# Found an rtlsdr! Attempt to extract bus and device number.
# Expecting something like: 'Bus 001 Device 005: ID 0bda:2838 Realtek Semiconductor Corp. RTL2838 DVB-T'
device_fields = device.split(' ')
# Attempt to cast fields to integers, to give some surety that we have the correct data.
device_bus = int(device_fields[1])
device_number = int(device_fields[3][:-1])
# Construct device address
reset_argument = '/dev/bus/usb/%03d/%03d' % (device_bus, device_number)
# Attempt to reset the device.
logging.info("Resetting device: %s" % reset_argument)
ret_code = subprocess.call(['./reset_usb', reset_argument])
logging.debug("Got return code: %s" % ret_code)
else:
continue
except:
logging.error("Errors occured while attempting to reset USB device.")
def sonde_search(config, attempts = 5):
""" Perform a frequency scan across the defined range, and test each frequency for a radiosonde's presence. """
@ -161,19 +196,19 @@ def sonde_search(config, attempts = 5):
while search_attempts > 0:
# Enable Bias-Tee if required.
if config['rtlsdr_bias']:
os.system('rtl_biast -b 1')
# Scan Band
run_rtl_power(config['min_freq']*1e6, config['max_freq']*1e6, config['search_step'], ppm=config['rtlsdr_ppm'], gain=config['rtlsdr_gain'])
run_rtl_power(config['min_freq']*1e6, config['max_freq']*1e6, config['search_step'], ppm=config['rtlsdr_ppm'], gain=config['rtlsdr_gain'], bias=config['rtlsdr_bias'])
# Read in result
try:
(freq, power, step) = read_rtl_power('log_power.csv')
except Exception as e:
traceback.print_exc()
logging.debug("Failed to read log_power.csv. Attempting to run rtl_power again.")
logging.error("Failed to read log_power.csv. Resetting RTLSDRs and attempting to run rtl_power again.")
# no log_power.csv usually means that rtl_power has locked up and had to be SIGKILL'd.
# This occurs when it can't get samples from the RTLSDR, because it's locked up for some reason.
# Issuing a USB Reset to the rtlsdr can sometimes solve this.
reset_rtlsdr()
search_attempts -= 1
time.sleep(10)
continue
@ -238,6 +273,13 @@ def process_rs_line(line):
rs_frame['id'] = str(params[1])
rs_frame['date'] = str(params[2])
rs_frame['time'] = str(params[3])
# Provide a clipped time field, without the milliseconds components.
# Do this just by splitting off everything before the '.', if it exists.
if '.' in rs_frame['time']:
rs_frame['short_time'] = rs_frame['time'].split('.')[0]
else:
rs_frame['short_time'] = rs_frame['time']
rs_frame['datetime_str'] = "%sT%s" % (rs_frame['date'], rs_frame['time'])
rs_frame['lat'] = float(params[4])
rs_frame['lon'] = float(params[5])
@ -250,7 +292,7 @@ def process_rs_line(line):
rs_frame['temp'] = 0.0
rs_frame['humidity'] = 0.0
logging.info("TELEMETRY: %s,%d,%s,%.5f,%.5f,%.1f" % (rs_frame['id'], rs_frame['frame'],rs_frame['time'], rs_frame['lat'], rs_frame['lon'], rs_frame['alt']))
logging.info("TELEMETRY: %s,%d,%s,%.5f,%.5f,%.1f,%s" % (rs_frame['id'], rs_frame['frame'],rs_frame['time'], rs_frame['lat'], rs_frame['lon'], rs_frame['alt'], rs_frame['crc']))
return rs_frame
@ -276,6 +318,8 @@ def update_flight_stats(data):
if data['alt'] > flight_stats['apogee']['alt']:
flight_stats['apogee'] = data
def calculate_flight_statistics():
""" Produce a flight summary, for inclusion in the log file. """
global flight_stats
@ -310,7 +354,7 @@ def calculate_flight_statistics():
def decode_rs92(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, almanac=None, ephemeris=None, timeout=120):
""" Decode a RS92 sonde """
global latest_sonde_data
global latest_sonde_data, internet_push_queue, ozi_push_queue
# Before we get started, do we need to download GPS data?
if ephemeris == None:
@ -328,7 +372,10 @@ def decode_rs92(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, a
logging.critical("Could not obtain GPS ephemeris or almanac data.")
return False
decode_cmd = "rtl_fm -p %d -M fm -s 12k -f %d 2>/dev/null |" % (int(ppm), frequency)
# Add a -T option if bias is enabled
bias_option = "-T " if bias else ""
decode_cmd = "rtl_fm %s-p %d -M fm -s 12k -f %d 2>/dev/null |" % (bias_option, int(ppm), frequency)
decode_cmd += "sox -t raw -r 12k -e s -b 16 -c 1 - -r 48000 -b 8 -t wav - lowpass 2500 highpass 20 2>/dev/null |"
# Note: I've got the check-CRC option hardcoded in here as always on.
@ -341,10 +388,6 @@ def decode_rs92(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, a
rx_last_line = time.time()
# Enable Bias-Tee if required.
if bias:
os.system('rtl_biast -b 1')
# Receiver subprocess. Discard stderr, and feed stdout into an asynchronous read class.
rx = subprocess.Popen(decode_cmd, shell=True, stdin=None, stdout=subprocess.PIPE, preexec_fn=os.setsid)
rx_stdout = AsynchronousFileReader(rx.stdout, autostart=True)
@ -366,7 +409,8 @@ def decode_rs92(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, a
if rx_queue != None:
try:
rx_queue.put_nowait(data)
internet_push_queue.put_nowait(data)
ozi_push_queue.put_nowait(data)
except:
pass
except:
@ -389,8 +433,11 @@ def decode_rs92(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, a
def decode_rs41(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, timeout=120):
""" Decode a RS41 sonde """
global latest_sonde_data
decode_cmd = "rtl_fm -p %d -M fm -s 12k -f %d 2>/dev/null |" % (int(ppm), frequency)
global latest_sonde_data, internet_push_queue, ozi_push_queue
# Add a -T option if bias is enabled
bias_option = "-T " if bias else ""
decode_cmd = "rtl_fm %s-p %d -M fm -s 12k -f %d 2>/dev/null |" % (bias_option, int(ppm), frequency)
decode_cmd += "sox -t raw -r 12k -e s -b 16 -c 1 - -r 48000 -b 8 -t wav - lowpass 2600 2>/dev/null |"
# Note: I've got the check-CRC option hardcoded in here as always on.
@ -400,10 +447,6 @@ def decode_rs41(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, t
rx_last_line = time.time()
# Enable Bias-Tee if required.
if bias:
os.system('rtl_biast -b 1')
# Receiver subprocess. Discard stderr, and feed stdout into an asynchronous read class.
rx = subprocess.Popen(decode_cmd, shell=True, stdin=None, stdout=subprocess.PIPE, preexec_fn=os.setsid)
rx_stdout = AsynchronousFileReader(rx.stdout, autostart=True)
@ -421,11 +464,14 @@ def decode_rs41(frequency, ppm=0, gain='automatic', bias=False, rx_queue=None, t
data['freq'] = "%.3f MHz" % (frequency/1e6)
data['type'] = "RS41"
update_flight_stats(data)
latest_sonde_data = data
if rx_queue != None:
try:
rx_queue.put_nowait(data)
internet_push_queue.put_nowait(data)
ozi_push_queue.put_nowait(data)
except:
pass
except:
@ -494,11 +540,40 @@ def internet_push_thread(station_config):
# Note that this will result in some odd upload times, due to leap seconds and otherwise, but should
# result in multiple stations (assuming local timezones are the same, and the stations are synced to NTP)
# uploading at roughly the same time.
while int(time.time())%config['upload_rate'] != 0:
while int(time.time())%station_config['upload_rate'] != 0:
time.sleep(0.1)
else:
# Otherwise, just sleep.
time.sleep(config['upload_rate'])
time.sleep(station_config['upload_rate'])
print("Closing thread.")
def ozi_push_thread(station_config):
""" Push a frame of sonde data into various internet services (APRS-IS, Habitat) """
global ozi_push_queue, OZI_PUSH_RUNNING
print("Started OziPlotter Push thread.")
while OZI_PUSH_RUNNING:
data = None
try:
# Wait until there is somethign in the queue before trying to process.
if ozi_push_queue.empty():
time.sleep(1)
continue
else:
# Read in entire contents of queue, and keep the most recent entry.
while not ozi_push_queue.empty():
data = ozi_push_queue.get()
except:
traceback.print_exc()
continue
try:
if station_config['ozi_enabled']:
push_telemetry_to_ozi(data,hostname=station_config['ozi_hostname'])
except:
traceback.print_exc()
time.sleep(station_config['ozi_update_rate'])
print("Closing thread.")
@ -533,7 +608,8 @@ if __name__ == "__main__":
timeout_time = time.time() + int(args.timeout)*60
# Internet push thread object.
push_thread = None
push_thread_1 = None
push_thread_2 = None
# Sonde Frequency & Type variables.
sonde_freq = None
@ -562,10 +638,14 @@ if __name__ == "__main__":
logging.info("Starting decoding of %s on %.3f MHz" % (sonde_type, sonde_freq/1e6))
# Start a thread to push data to the web, if it isn't started already.
if push_thread == None:
push_thread = Thread(target=internet_push_thread, kwargs={'station_config':config})
push_thread.start()
# Start both of our internet/ozi push threads.
if push_thread_1 == None:
push_thread_1 = Thread(target=internet_push_thread, kwargs={'station_config':config})
push_thread_1.start()
if push_thread_2 == None:
push_thread_2 = Thread(target=ozi_push_thread, kwargs={'station_config':config})
push_thread_2.start()
# Start decoding the sonde!
if sonde_type == "RS92":

Wyświetl plik

@ -6,6 +6,7 @@
# Build rs_detect.
cd ../scan/
gcc rs_detect.c -lm -o rs_detect
gcc reset_usb.c -o reset_usb
# Build rs92 and rs41 decoders
@ -23,6 +24,7 @@ gcc rs_main92.o rs_rs92.o rs_bch_ecc.o rs_demod.o rs_datum.o -lm -o rs92mod
# Copy all necessary files into this directory.
cd ../auto_rx/
cp ../scan/rs_detect .
cp ../scan/reset_usb .
cp ../rs_module/rs41mod .
cp ../rs_module/rs92mod .

Wyświetl plik

@ -34,7 +34,10 @@ def read_auto_rx_config(filename):
'payload_callsign': 'RADIOSONDE',
'uploader_callsign': 'SONDE_AUTO_RX',
'uploader_lat' : 0.0,
'uploader_lon' : 0.0
'uploader_lon' : 0.0,
'ozi_enabled' : False,
'ozi_update_rate': 5,
'ozi_hostname' : '127.0.0.1'
}
try:
@ -65,6 +68,9 @@ def read_auto_rx_config(filename):
auto_rx_config['uploader_callsign'] = config.get('habitat', 'uploader_callsign')
auto_rx_config['uploader_lat'] = config.getfloat('habitat', 'uploader_lat')
auto_rx_config['uploader_lon'] = config.getfloat('habitat', 'uploader_lon')
auto_rx_config['ozi_enabled'] = config.getboolean('oziplotter', 'ozi_enabled')
auto_rx_config['ozi_update_rate'] = config.getint('oziplotter', 'ozi_update_rate')
auto_rx_config['ozi_hostname'] = config.get('oziplotter', 'ozi_hostname')
return auto_rx_config

Wyświetl plik

@ -0,0 +1,59 @@
# OziPlotter push utils for Sonde auto RX.
import socket
import json
# Network Settings
HORUS_UDP_PORT = 55672
HORUS_OZIPLOTTER_PORT = 8942
# Send an update on the core payload telemetry statistics into the network via UDP broadcast.
# This can be used by other devices hanging off the network to display vital stats about the payload.
def send_payload_summary(callsign, latitude, longitude, altitude, speed=-1, heading=-1):
packet = {
'type' : 'PAYLOAD_SUMMARY',
'callsign' : callsign,
'latitude' : latitude,
'longitude' : longitude,
'altitude' : altitude,
'speed' : speed,
'heading': heading
}
# Set up our UDP socket
s = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
s.settimeout(1)
# Set up socket for broadcast, and allow re-use of the address
s.setsockopt(socket.SOL_SOCKET,socket.SO_BROADCAST,1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except:
pass
s.bind(('',HORUS_UDP_PORT))
try:
s.sendto(json.dumps(packet), ('<broadcast>', HORUS_UDP_PORT))
except socket.error:
s.sendto(json.dumps(packet), ('127.0.0.1', HORUS_UDP_PORT))
# The new 'generic' OziPlotter upload function, with no callsign, or checksumming (why bother, really)
def oziplotter_upload_basic_telemetry(time, latitude, longitude, altitude, hostname="192.168.88.2"):
sentence = "TELEMETRY,%s,%.5f,%.5f,%d\n" % (time, latitude, longitude, altitude)
try:
ozisock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
ozisock.sendto(sentence,(hostname,HORUS_OZIPLOTTER_PORT))
ozisock.close()
except Exception as e:
print("Failed to send to Ozi: " % e)
# Call both of the above functions, with radiosonde telemetry data.
def push_telemetry_to_ozi(telemetry, hostname='127.0.0.1'):
# Payload data summary.
send_payload_summary(telemetry['id'], telemetry['lat'], telemetry['lon'], telemetry['alt'])
# Telemetry to OziPlotter
oziplotter_upload_basic_telemetry(telemetry['short_time'], telemetry['lat'], telemetry['lon'], telemetry['alt'], hostname=hostname)

Wyświetl plik

@ -41,7 +41,7 @@ upload_rate = 30
# Upload when (seconds_since_utc_epoch%upload_rate) == 0. Otherwise just delay upload_rate seconds between uploads.
# Setting this to True with multple uploaders should give a higher chance of all uploaders uploading the same frame,
# however the upload_rate should not be set too low, else there may be a chance of missing upload slots.
synchronous_upload = False
synchronous_upload = True
# Enable upload to various services.
enable_aprs = False
enable_habitat = False
@ -80,4 +80,9 @@ uploader_callsign = SONDE_AUTO_RX
uploader_lat = 0.0
uploader_lon = 0.0
# Settings for pushing data into OziPlotter
# Oziplotter receives data via a basic CSV format, via UDP.
[oziplotter]
ozi_enabled = False
ozi_update_rate = 5
ozi_hostname = 127.0.0.1