radiosonde_auto_rx/auto_rx/utils/plot_sonde_log.py

493 wiersze
18 KiB
Python

#!/usr/bin/env python
#
# Radiosonde Log Plotter
#
# Copyright (C) 2019 Mark Jessop <vk5qi@rfhead.net>
# Released under GNU GPL v3 or later
#
# Note: This script is very much a first pass, and doesn't have much error checking of data.
#
# Dependencies:
# numpy
# pytz
# dateutil
# metpy
# You should be able to get the above with either your system package manager, or Pip.
# I would strongly suggest running this under Python 3.5 or newer.
#
#
# There are two general usage scenarios, plotting a single file, and plotting an entire directory.
#
# Single file plotting:
# $ python plot_sonde_log.py --singlefile 20190424-105731_P4750324_RS41_401500_sonde.log
#
# Plotting of a directory of files.
# In this scenario we need to supply the following parameters:
# --log-dir The directory containing the sonde log files (usually radiosonde_auto_rx/auto_rx/log/)
# --output-dir Where to save the plots to.
#
# A file called plot_status.txt will be created, which will keep track of which log files have already been
# completely processed. Log files will be re-processed until:
# - The sonde is detected to have burst.
# - The last position is more than 15 minutes old.
# Additionally, log files will not be processed if:
# - They contain less than 500 positions.
# - The first observed altitude is > 5km (indicative of a far-away sonde.)
#
# Example call:
# # python plot_sonde_log.py --log-dir=/home/pi/radiosonde_auto_rx/auto_rx/log/ --output-dir=/home/pi/soundings/
#
# This can be called from a bash script, run by a cron-job. For example, create a file ~/generate_soundings.sh
# containing the following:
#
# #!/bin/bash
# # Generate Soundings
# python plot_sonde_log.py --log-dir=/home/pi/radiosonde_auto_rx/auto_rx/log/ --output-dir=/home/pi/soundings/
# # Copy to another server (SSH keys would need to be setup for this to work)
# rsync -r /home/pi/soundings/ yourserver:~/path/to/soundings/
#
# A cron-job could then be set up with the comamnd:
# */20 23,0,1,2,11,12,13,14 * * * /home/pi/generate_soundings.sh
#
# This will run the above script every 20 minutes during the hours when we expect to see 00Z and 12Z sondes.
# NOTE: You will likely need to uncomment the two lines identified below to be able to run this on a headless
# Raspberry Pi.
import argparse
import datetime
import glob
import json
import os.path
import pytz
import sys
import traceback
import numpy as np
# NOTE - If running on a headless system with no display, the following two lines will need to be uncommented.
#import matplotlib as mpl
#mpl.use('Agg')
import matplotlib.pyplot as plt
from dateutil.parser import parse
from math import radians, degrees, sin, cos, atan2, sqrt, pi
import metpy.calc as mpcalc
from metpy.plots import SkewT
from metpy.units import units
# Earthmaths code by Daniel Richman (thanks!)
# Copyright 2012 (C) Daniel Richman; GNU GPL 3
def position_info(listener, balloon):
"""
Calculate and return information from 2 (lat, lon, alt) tuples
Returns a dict with:
- angle at centre
- great circle distance
- distance in a straight line
- bearing (azimuth or initial course)
- elevation (altitude)
Input and output latitudes, longitudes, angles, bearings and elevations are
in degrees, and input altitudes and output distances are in meters.
"""
# Earth:
radius = 6371000.0
(lat1, lon1, alt1) = listener
(lat2, lon2, alt2) = balloon
lat1 = radians(lat1)
lat2 = radians(lat2)
lon1 = radians(lon1)
lon2 = radians(lon2)
# Calculate the bearing, the angle at the centre, and the great circle
# distance using Vincenty's_formulae with f = 0 (a sphere). See
# http://en.wikipedia.org/wiki/Great_circle_distance#Formulas and
# http://en.wikipedia.org/wiki/Great-circle_navigation and
# http://en.wikipedia.org/wiki/Vincenty%27s_formulae
d_lon = lon2 - lon1
sa = cos(lat2) * sin(d_lon)
sb = (cos(lat1) * sin(lat2)) - (sin(lat1) * cos(lat2) * cos(d_lon))
bearing = atan2(sa, sb)
aa = sqrt((sa ** 2) + (sb ** 2))
ab = (sin(lat1) * sin(lat2)) + (cos(lat1) * cos(lat2) * cos(d_lon))
angle_at_centre = atan2(aa, ab)
great_circle_distance = angle_at_centre * radius
# Armed with the angle at the centre, calculating the remaining items
# is a simple 2D triangley circley problem:
# Use the triangle with sides (r + alt1), (r + alt2), distance in a
# straight line. The angle between (r + alt1) and (r + alt2) is the
# angle at the centre. The angle between distance in a straight line and
# (r + alt1) is the elevation plus pi/2.
# Use sum of angle in a triangle to express the third angle in terms
# of the other two. Use sine rule on sides (r + alt1) and (r + alt2),
# expand with compound angle formulae and solve for tan elevation by
# dividing both sides by cos elevation
ta = radius + alt1
tb = radius + alt2
ea = (cos(angle_at_centre) * tb) - ta
eb = sin(angle_at_centre) * tb
elevation = atan2(ea, eb)
# Use cosine rule to find unknown side.
distance = sqrt((ta ** 2) + (tb ** 2) - 2 * tb * ta * cos(angle_at_centre))
# Give a bearing in range 0 <= b < 2pi
if bearing < 0:
bearing += 2 * pi
return {
"listener": listener, "balloon": balloon,
"listener_radians": (lat1, lon1, alt1),
"balloon_radians": (lat2, lon2, alt2),
"angle_at_centre": degrees(angle_at_centre),
"angle_at_centre_radians": angle_at_centre,
"bearing": degrees(bearing),
"bearing_radians": bearing,
"great_circle_distance": great_circle_distance,
"straight_distance": distance,
"elevation": degrees(elevation),
"elevation_radians": elevation
}
def read_log_file(filename, decimation=10, min_altitude=100):
# Load in the file.
# data = np.genfromtxt(filename,delimiter=',', dtype=None)
# # Extract fields.
# times = data['f0']
# latitude = data['f3']
# longitude = data['f4']
# altitude = data['f5']
# temperature = data['f6']
# humidity = data['f7']
times = []
latitude = []
longitude = []
altitude = []
temperature = []
humidity = []
snr = []
ferror = []
with open(filename, 'r') as _file:
for line in _file:
try:
_fields = line.split(',')
# Log fields: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
# "timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,type,freq_mhz,snr,f_error_hz,sats,batt_v,burst_timer,aux_data\n"
# Attempt to parse the line
_time = _fields[0]
_lat = float(_fields[3])
_lon = float(_fields[4])
_alt = float(_fields[5])
_temp = float(_fields[9])
_hum = float(_fields[10])
try:
# Attempt to extract SNR and frequency error fields.
# These may not be present on older log files.
_snr = float(_fields[13])
_ferror = float(_fields[14])
except:
_snr = -99
_ferror = 0.0
# Append data to arrays.
times.append(_time)
latitude.append(_lat)
longitude.append(_lon)
altitude.append(_alt)
temperature.append(_temp)
humidity.append(_hum)
snr.append(_snr)
ferror.append(_ferror)
except Exception as e:
print("Error reading line: ")
print("Read %d data points from %s." % (len(times), filename))
_output = [] # Altitude, Wind Speed, Wind Direction, Temperature, Dew Point
# First entry, We assume all the values are unknown for now.
_output.append([altitude[0], np.NaN, np.NaN, np.NaN, np.NaN, np.NaN, snr[0], ferror[0]])
_burst = False
_startalt = altitude[0]
i = decimation
while i < len(times):
if altitude[i] < min_altitude:
i += decimation
continue
# Check if we are descending. If so, break.
if altitude[i] < _output[-1][0]:
_burst = True
print("Detected burst at %d metres." % altitude[i])
break
# If we have valid PTU data, calculate the dew point.
if temperature[i] != -273:
T = temperature[i]
RH = humidity[i]
DP = 243.04*(np.log(RH/100)+((17.625*T)/(243.04+T)))/(17.625-np.log(RH/100)-((17.625*T)/(243.04+T)))
else:
# Otherwise we insert NaNs, so data isn't plotted.
T = np.NaN
DP = np.NaN
RH = np.NaN
# Calculate time delta between telemetry frames.
_time = parse(times[i])
_time_old = parse(times[i-decimation])
_delta_seconds = (_time - _time_old).total_seconds()
# Calculate the movement direction and distance, and then calculate the movement speed.
_movement = position_info((latitude[i], longitude[i], altitude[i]), (latitude[i-decimation], longitude[i-decimation], altitude[i-decimation]))
_heading = _movement['bearing']
_velocity = _movement['great_circle_distance']/_delta_seconds
_output.append([altitude[i], _velocity, _heading, T, DP, RH, snr[i], ferror[i]])
i += decimation
# Convert our output data into something we can process easier.
return (np.array(_output), _burst, _startalt, times[-1], snr, ferror)
def plot_matplotlib(data_np, title="", metric=False, alt_limit = 20000, temp_limit=None):
if metric:
_alt = data_np[:,0]
else:
_alt = data_np[:,0]*3.28084 # Convert to feet.
_speed = data_np[:,1]
_direction = data_np[:,2]/10.0
_temp = data_np[:,3]
_dp = data_np[:,4]
# Produce a boolean array to limit the plotted data.
_data_limit = _alt < alt_limit
# Plot the data...
plt.figure()
plt.plot(_speed[_data_limit], _alt[_data_limit], label='Speed (kt)', color='g')
plt.plot(_direction[_data_limit], _alt[_data_limit], label='Direction (deg/10)', color='m')
plt.plot(_temp[_data_limit], _alt[_data_limit], label='Temp (deg C)', color='b')
plt.plot(_dp[_data_limit], _alt[_data_limit], label='DP (deg C)', color='r')
if metric:
plt.ylabel("Altitude (metres)")
else:
plt.ylabel("Altitude (feet)")
# Determine and set plot axis limits
_axes = plt.gca()
# Y limit is either a default value, or a user specified altitude.
_axes.set_ylim(top=alt_limit, bottom=0)
# X limits are based on a combination of data.
# The upper limit is based on the maximum speed within our altitude window
if temp_limit == None:
_temp_in_range= _temp[_data_limit]
_dp_in_range= _dp[_data_limit]
_min_temp = np.min(_temp_in_range[~np.isnan(_temp_in_range)])
_min_dp = np.min(_dp_in_range[~np.isnan(_dp_in_range)])
_axes.set_xlim(left=min(_min_temp, _min_dp))
else:
_axes.set_xlim(left=temp_limit)
plt.title("Sounding File: %s" % title)
plt.grid(which='both')
plt.legend(loc='upper right')
plt.show()
def plot_metpy(data, title="", saveplot=None, showplot=True):
# Convert data into a suitable format for metpy.
_altitude = data[:,0] * units('m')
p = mpcalc.height_to_pressure_std(_altitude)
T = data[:,3] * units.degC
Td = data[:,4] * units.degC
wind_speed = data[:,1] * units('m/s')
wind_direction = data[:,2] * units.degrees
u, v = mpcalc.wind_components(wind_speed, wind_direction)
fig = plt.figure(figsize=(6,8))
skew = SkewT(fig=fig)
skew.plot(p, T, 'r')
skew.plot(p, Td, 'g')
my_interval = np.arange(300, 1000, 50) * units('mbar')
ix = mpcalc.resample_nn_1d(p, my_interval)
skew.plot_barbs(p[ix], u[ix], v[ix])
skew.ax.set_ylim(1000,300)
skew.ax.set_xlim(-40, 30)
skew.plot_dry_adiabats()
heights = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) * units.km
std_pressures = mpcalc.height_to_pressure_std(heights)
for height_tick, p_tick in zip(heights, std_pressures):
trans, _, _ = skew.ax.get_yaxis_text1_transform(0)
skew.ax.text(0.02, p_tick, '---{:~d}'.format(height_tick), transform=trans)
plt.title("Sounding: " + title)
if saveplot != None:
fig.savefig(saveplot, bbox_inches='tight')
#
# Status file handling.
# The status file contains a JSON blob with one entry per filename that has been opened.
# Each entry contains if the flight is considered to be 'finished', which is when either
# the payload has started to descend, or no data has been received for ~10 min.
#
def read_status_file(filename):
# Check the file exists..
if not os.path.isfile(filename):
# File does not exist, create a blank one.
write_status_file(filename, {})
# Now open and read the file.
try:
_f = open(filename,'r')
data = json.loads(_f.read())
_f.close()
return data
except Exception as e:
print("Error reading status file - %s" % str(e))
return None
def write_status_file(filename, data):
_f = open(filename, 'w')
_f.write(json.dumps(data))
_f.close()
def process_directory(log_dir, output_dir, status_file, time_limit = 60):
# Load the status file.
_log_status = read_status_file(status_file)
if _log_status is None:
return
# Get a list of log files in the directory.
_files = glob.glob(os.path.join(log_dir, "*_sonde.log"))
for _file in _files:
_basename = os.path.basename(_file)
# Check if we have touched this file before.
if _basename in _log_status:
if _log_status[_basename]['complete']:
print("Already finished processing %s" % _basename)
continue
else:
# This file needs to be re-processed
pass
else:
# Add an entry for this file.
_log_status[_basename] = {'complete': False}
# Read in the file!
try:
(data, burst, startalt, last_time, snr, ferror) = read_log_file(_file, decimation=10)
# Don't process files with a starting altitude well above ground.
# This indicates it's likely a sonde from a long way away.
if startalt > 2000:
_log_status[_basename]['complete'] = True
print("Not processing %s." % _basename)
continue
# Calculate the age of the last data point in minutes.
_data_age = (datetime.datetime.now(datetime.timezone.utc) - parse(last_time)).total_seconds() / 60.0
if burst or (_data_age > time_limit):
# We consider this file to be finished.
_log_status[_basename]['complete'] = True
# Plot the data, and save to disk.
_out_file = os.path.join(output_dir, _basename[:-4]+".png")
_file_timestamp = _basename.split('_')[0]
_sonde_serial = _basename.split('_')[1]
_title = _file_timestamp + " " + _sonde_serial
print("Generating plot for: %s" % _basename)
plot_metpy(data, title=_title, saveplot=_out_file, showplot=False)
except Exception as e:
traceback.print_exc()
print("Error processing file %s - %s" % (_basename, str(e)))
# Write out the status file
write_status_file(status_file, _log_status)
if __name__ == "__main__":
# Data format:
# 2019-04-17T00:40:40.000Z,P4740856,7611,-35.38981,139.47062,12908.1,-67.9,25.0,RS41,402.500,SATS 9,BATT -1.0
parser = argparse.ArgumentParser()
parser.add_argument("--singlefile", default = "", type=str, help="Single log file to process.")
parser.add_argument("--metric", action="store_true", default=False, help="Use metric altitudes. (Default is to use Feet)")
parser.add_argument("--alt-limit", default=20000, type=int, help="Limit plot to supplied altitude (feet or metres, depending on user selection)")
parser.add_argument("--temp-limit", default=None, type=float, help="Limit plot to a lower temperature in degrees. (Default is no limit, plot will autoscale)")
parser.add_argument("--decimation", default=10, type=int, help="Decimate input data by X times. (Default = 10)")
parser.add_argument("--log-dir", default="../log/", type=str, help="Directory containing sonde logs to process.")
parser.add_argument("--output-dir", default="./plots/", type=str, help="Output directory to save plots to.")
parser.add_argument("--plot-status-file", default="plot_status.txt", type=str, help="Plotting status file.")
parser.add_argument("--snr", default=False, action='store_true', help="Plot SNR vs time.")
parser.add_argument("--ferror", default=False, action='store_true', help="Plot Frequency Error vs time.")
args = parser.parse_args()
if args.singlefile != "":
# Process a single file.
(data_np, burst, startalt, last_time, snr, ferror) = read_log_file(args.singlefile, decimation=args.decimation)
#plot_matplotlib(data_np, title=os.path.basename(args.filename), metric=args.metric, alt_limit=args.alt_limit, temp_limit=args.temp_limit)
plot_metpy(data_np, saveplot=None, showplot=True)
if args.snr:
plt.figure()
plt.plot(snr)
plt.xlabel("Sample")
plt.ylabel("SNR (dB)")
plt.title("SNR vs Sample")
plt.grid()
if args.ferror:
plt.figure()
plt.plot(ferror)
plt.xlabel("Sample")
plt.ylabel("Frequency Error (Hz)")
plt.title("Frequency Error vs Sample")
plt.grid()
plt.show()
else:
# do a batch process run.
process_directory(args.log_dir, args.output_dir, args.plot_status_file)