kopia lustrzana https://github.com/sm3ulc/hab-wspr
Big update with timeslots, loggging, command line options
sm3ulc
rodzic
75e7cb406b
commit
589787e48a
59
README.md
59
README.md
|
|
@ -7,60 +7,83 @@ The software webscrapes data from wsprnet.org, filter out calls from the balloon
|
|||
* habhub tracker ( https://tracker.habhub.org/ )
|
||||
* aprs-fi ( https://aprs.fi ).
|
||||
|
||||
There are existing functions to load/save flightdata from csv/wsprnet-archive-files. ( http://wsprnet.org/drupal/downloads )
|
||||
There are existing functions to load/save flightdata from csv/wsprnet-archive-files.
|
||||
( http://wsprnet.org/drupal/downloads )
|
||||
|
||||
The protocol for the telemetry is described here:
|
||||
|
||||
* https://qrp-labs.com/flights/s4.html
|
||||
|
||||
|
||||
# Installation
|
||||
|
||||
First clone the repo:
|
||||
|
||||
git clone https://github.com/sm3ulc/hab-wspr
|
||||
<pre>
|
||||
git clone https://github.com/sm3ulc/hab-wspr
|
||||
</pre>
|
||||
|
||||
The package requires some extra modules that need to be installed via pip or similar
|
||||
|
||||
apt install python-httplib2 python-requests python3-bs4
|
||||
|
||||
<pre>
|
||||
apt install python-httplib2 python-requests python3-bs4
|
||||
</pre>
|
||||
|
||||
|
||||
For windows users install anaconda with python 3.
|
||||
|
||||
<pre>
|
||||
pip install httplib2
|
||||
pip install bs4
|
||||
</pre>
|
||||
|
||||
# Configuration
|
||||
|
||||
Edit balloon.ini and add aprs-is user etc. Add balloons on the format:
|
||||
|
||||
[ habhub name, ham callsign for the balloon , band in mhz, channel ]
|
||||
<pre>
|
||||
[ habhub name, ham callsign for the balloon , band in mhz, channel, timeslot ]
|
||||
</pre>
|
||||
|
||||
timeslot = 0 to disable use of timeslots. 1-5, use correspondent slot 00, 02, 04 etc.
|
||||
|
||||
|
||||
Uploads to APRS-IS is done by adding the SSID "-12" to the default balloon-callsign.
|
||||
|
||||
|
||||
To run:
|
||||
To run on linux: (with default config file balloon.ini)
|
||||
|
||||
python3 webscrape.py
|
||||
<pre>
|
||||
python3 webscrape.py
|
||||
<pre>
|
||||
|
||||
|
||||
The scripts work with a database in sqlite. It can be used to do all kinds of output/export like checking the last sent spots:
|
||||
|
||||
sqlite3 wsprdb.db 'select * from sentspots order by time_sent desc limit 30'
|
||||
<pre>
|
||||
sqlite3 wsprdb.db 'select * from sentspots order by time_sent desc limit 30'
|
||||
</pre>
|
||||
|
||||
# Testing
|
||||
|
||||
Adjust your balloon.ini
|
||||
Adjust your balloon.ini or other configfile like test.ini.
|
||||
|
||||
Goto http://wsprnet.org/drupal/downloads or
|
||||
|
||||
<pre>
|
||||
wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz
|
||||
</pre>
|
||||
|
||||
wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz
|
||||
|
||||
Extract data from archive and save filtered spots to spots.csv and then process.
|
||||
|
||||
python3 webscrape.py --archive wsprspots-2019-12.csv.gz --conf test.ini
|
||||
Extract data from archive and append filtered spots to spots.csv in and then process.
|
||||
|
||||
<pre>
|
||||
python3 webscrape.py --archive wsprspots-2019-12.csv.gz --conf test.ini
|
||||
</pre>
|
||||
|
||||
Read csv-file from spots.csv and process.
|
||||
|
||||
<pre>python3 webscrape.py
|
||||
|
||||
--csv spots.csv
|
||||
|
||||
<pre>
|
||||
python3 webscrape.py --csv spots.csv
|
||||
</pre>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,5 +11,8 @@ aprsUser = 'myaprsisuser'
|
|||
# APRS-IS passcode for your callsign.
|
||||
aprsPass = 'myaprsispass'
|
||||
|
||||
# [ habhub name, aprs-call, band in mhz, channel ]
|
||||
balloons = [[ "BSY22","SA7XXX",14,11]]
|
||||
# [ habhub name, aprs-call, band in mhz, channel, timeslot ]
|
||||
balloons = [[ "BSY22","SA7XXX",14,11,0 ],
|
||||
[ "B2", "SOMECALL",14,12,0 ]]
|
||||
|
||||
|
||||
|
|
|
|||
55
balloon.py
55
balloon.py
|
|
@ -1,6 +1,7 @@
|
|||
|
||||
# from datetime import datetime,timedelta
|
||||
import datetime
|
||||
import logging
|
||||
import sqlite3
|
||||
import csv
|
||||
import sys
|
||||
|
|
@ -22,9 +23,9 @@ def balloonstodb(balloons):
|
|||
if not data:
|
||||
con.commit()
|
||||
except sqlite3.Error as e:
|
||||
print("Database error: %s" % e)
|
||||
logging.info("Database error: %s",e)
|
||||
except Exception as e:
|
||||
print("Exception in _query: %s" % e)
|
||||
logging.info("Exception in _query:", e)
|
||||
finally:
|
||||
if con:
|
||||
con.close()
|
||||
|
|
@ -60,6 +61,7 @@ def readballoonsdb():
|
|||
|
||||
# Dumps all spots to csv-file
|
||||
def dumpcsv(spotlist):
|
||||
logging.info("Writing spots to csv-file")
|
||||
with open('spots.csv', 'a', newline='') as csvfile:
|
||||
spotswriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
|
||||
|
||||
|
|
@ -68,9 +70,10 @@ def dumpcsv(spotlist):
|
|||
spotswriter.writerow(row)
|
||||
|
||||
csvfile.close()
|
||||
return
|
||||
|
||||
|
||||
|
||||
# Org-csv
|
||||
# Date Call Frequency SNR Drift Grid dBm W reporter locator dist-km dist-mi
|
||||
# 2018-05-21 19:04,F6HCO,10.140216,-11,1,J19bg,+33,1.995,SM0EPX/RX2,JO89si,1495,929
|
||||
|
||||
|
|
@ -80,29 +83,33 @@ def dumpcsv(spotlist):
|
|||
# 2018-06-14 09:08,QA5IQB,5.288761,-26,0,JO22,+20,DL0HT,JO43jb,266
|
||||
# 0 1 2 3 4 5 6 7 8 9
|
||||
|
||||
# Local-csv
|
||||
# 2018-05-01 02:14,QA5IQA,3.594159,-28,JO53,27,DC5AL-R,JO31lk,353
|
||||
# 2018-05-01 02:14,QA5IQA,3.594153,-21,JO53,27,DF2JP,JO31hh,380
|
||||
|
||||
# timestamp, tx_call, freq real, snr integer, drift integer, tx_loc, power , rx_call, rx_loc, distance
|
||||
|
||||
def readcsv():
|
||||
spots = []
|
||||
with open('spots.csv', newline='') as csvfile:
|
||||
spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
|
||||
spots = []
|
||||
with open('spots.csv', newline='') as csvfile:
|
||||
spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
|
||||
|
||||
for row in spotsreader:
|
||||
for row in spotsreader:
|
||||
# Time
|
||||
row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
|
||||
row[3] = int(row[3])
|
||||
row[4] = int(row[4])
|
||||
|
||||
# Strip "+" from dB
|
||||
row[6] = int(row[6].replace('+',''))
|
||||
row[9] = int(row[9])
|
||||
|
||||
# print(row)
|
||||
spots.append(row)
|
||||
|
||||
csvfile.close()
|
||||
logging.info("Loaded spots: %s", len(spots))
|
||||
#print("First",spots[1:][0])
|
||||
#print("Last",spots[-1:][0])
|
||||
|
||||
row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
|
||||
row[3] = int(row[3])
|
||||
row[4] = int(row[4])
|
||||
# Strip "+" from dB
|
||||
row[6] = int(row[6].replace('+',''))
|
||||
row[9] = int(row[9])
|
||||
|
||||
# print(row)
|
||||
spots.append(row)
|
||||
|
||||
csvfile.close()
|
||||
# print(spotsreader)
|
||||
print("Loaded spots:", len(spots))
|
||||
print("First",spots[1:][0])
|
||||
print("Last",spots[-1:][0])
|
||||
|
||||
return spots
|
||||
return spots
|
||||
|
|
|
|||
|
|
@ -18,35 +18,30 @@
|
|||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
|
||||
import configparser
|
||||
import time, datetime, urllib3, sys
|
||||
# from pykml import parser
|
||||
from socket import *
|
||||
|
||||
# APRS-IS login info
|
||||
serverHost = 'euro.aprs2.net' # Pick a local server if you like
|
||||
serverPort = 14580
|
||||
|
||||
aprsUser = 'xxxx' # Replace with your own callsign. This should NOT have any SSID's on it (i.e. -9)
|
||||
aprsPass = '12345' # APRS-IS passcode for your callsign.
|
||||
config = configparser.ConfigParser()
|
||||
config.read('balloon.ini')
|
||||
aprsUser = config['main']['aprsUser']
|
||||
aprsPass = config['main']['aprsPass']
|
||||
|
||||
# APRS packet Settings
|
||||
callsign = aprsUser # This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login.
|
||||
callsign = "xxxx"
|
||||
|
||||
# aprsUser # This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login.
|
||||
|
||||
update_rate = 30 # Update rate, in seconds.
|
||||
# This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login.
|
||||
callsign = config['main']['aprsCallsign']
|
||||
|
||||
# Get KML from SondeMonitor and parse into a Python dictionary
|
||||
def get_sonde():
|
||||
sonde_data = {}
|
||||
|
||||
sonde_data["lat"] = str(sys.argv[2])
|
||||
sonde_data["lon"] = str(sys.argv[3])
|
||||
sonde_data["alt"] = "10000"
|
||||
sonde_data["id"] = str(sys.argv[1])
|
||||
|
||||
return sonde_data
|
||||
|
||||
# Push a Radiosonde data packet to APRS as an object.
|
||||
|
|
@ -82,7 +77,7 @@ def push_balloon_to_aprs(sonde_data):
|
|||
alt = int(float(sonde_data["alt"])/0.3048)
|
||||
|
||||
# Produce the APRS object string.
|
||||
out_str = ";%s*111111z%s/%sO000/000/A=%06d Ballooon" % (object_name,lat_str,lon_str,alt)
|
||||
out_str = ";%s*111111z%s/%sO000/000/A=%06d Balloon" % (object_name,lat_str,lon_str,alt)
|
||||
print(out_str)
|
||||
|
||||
# Connect to an APRS-IS server, login, then push our object position in.
|
||||
|
|
|
|||
219
telemetry.py
219
telemetry.py
|
|
@ -6,7 +6,9 @@ import csv
|
|||
import datetime
|
||||
from datetime import datetime,timedelta
|
||||
import gzip
|
||||
import hashlib
|
||||
import httplib2
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
|
|
@ -14,6 +16,8 @@ import sqlite3
|
|||
import sys
|
||||
import time
|
||||
|
||||
from pprint import pformat
|
||||
|
||||
import maidenhead
|
||||
|
||||
from balloon import *
|
||||
|
|
@ -45,14 +49,6 @@ def trim(spots):
|
|||
|
||||
return spots
|
||||
|
||||
#a = list(range(10))
|
||||
#a.reverse()
|
||||
#print(a)
|
||||
#a = trim(a)
|
||||
#print(a)
|
||||
#sys.exit(0)
|
||||
|
||||
|
||||
# Read new spots from database
|
||||
def readnewspotsdb():
|
||||
spots = []
|
||||
|
|
@ -112,8 +108,9 @@ def readnewspotsdb():
|
|||
|
||||
|
||||
def readgz(balloons, gzfile):
|
||||
print("Reading gz:", gzfile)
|
||||
|
||||
logging.info("Reading gz: %s", gzfile)
|
||||
|
||||
rows = 0
|
||||
spots = []
|
||||
calls = []
|
||||
for b in balloons:
|
||||
|
|
@ -123,29 +120,41 @@ def readgz(balloons, gzfile):
|
|||
spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
|
||||
|
||||
for row in spotsreader:
|
||||
rows += 1
|
||||
# print(row)
|
||||
# Select correct fields in correct order
|
||||
row = [row[1], row[6], row[5], row[4], row[9], row[7], row[8],row[2],row[3],row[10]]
|
||||
|
||||
# print(', '.join(row))
|
||||
for c in calls:
|
||||
# print("Found", c, row)
|
||||
if row[1] == c:
|
||||
# Remove selfmade WSPR tranmissions
|
||||
if len(row[5]) == 4:
|
||||
row[0] = datetime.datetime.fromtimestamp(int(row[0]))
|
||||
# print("Found", c, row)
|
||||
row[3] = int(row[3])
|
||||
row[4] = int(row[4])
|
||||
|
||||
# Strip "+" from dB
|
||||
row[6] = int(row[6].replace('+',''))
|
||||
row[9] = int(row[9])
|
||||
# print("Found", c, row)
|
||||
spots.append(row)
|
||||
|
||||
if re.match('(^0|^Q).[0-9].*', row[1]):
|
||||
row[0] = datetime.datetime.fromtimestamp(int(row[0]))
|
||||
# print("Found", row)
|
||||
|
||||
row[3] = int(row[3])
|
||||
row[4] = int(row[4])
|
||||
|
||||
# Strip "+" from dB
|
||||
row[6] = int(row[6].replace('+',''))
|
||||
row[9] = int(row[9])
|
||||
|
||||
spots.append(row)
|
||||
# sys.exit(0)
|
||||
|
||||
print("Nr-calls+telem:", len(spots))
|
||||
|
||||
print("Total rows", rows, "Nr-calls+telem:", len(spots))
|
||||
csvfile.close()
|
||||
print("Done reading")
|
||||
|
||||
return spots
|
||||
|
||||
|
|
@ -182,28 +191,20 @@ def decode_telemetry(spot_pos, spot_tele):
|
|||
c1 = spot_tele_call[1]
|
||||
# print("C1=",c1)
|
||||
if c1.isalpha():
|
||||
# print("c1=alpha")
|
||||
c1=ord(c1)-55
|
||||
# print("new c1=",c1)
|
||||
else:
|
||||
c1=ord(c1)-48
|
||||
|
||||
# print("new c1=",c1)
|
||||
|
||||
c2=ord(spot_tele_call[3])-65
|
||||
c3=ord(spot_tele_call[4])-65
|
||||
c4=ord(spot_tele_call[5])-65
|
||||
|
||||
# print("C:",c1,c2,c3,c4)
|
||||
|
||||
# Convert locator to numbers
|
||||
l1=ord(spot_tele_loc[0])-65
|
||||
l2=ord(spot_tele_loc[1])-65
|
||||
l3=ord(spot_tele_loc[2])-48
|
||||
l4=ord(spot_tele_loc[3])-48
|
||||
|
||||
# print("L:",l1,l2,l3,l4)
|
||||
|
||||
#
|
||||
# Convert power
|
||||
#
|
||||
|
|
@ -249,8 +250,6 @@ def decode_telemetry(spot_pos, spot_tele):
|
|||
lsub2=lsub2+65
|
||||
subloc=(chr(lsub1)+chr(lsub2)).lower()
|
||||
|
||||
# print("alt:",alt,"subloc:", lsub1, lsub2_tmp, lsub2,subloc)
|
||||
|
||||
# Temperature
|
||||
# 40*42*2*2
|
||||
temp_1=int(sum2_tot/6720)
|
||||
|
|
@ -282,7 +281,7 @@ def decode_telemetry(spot_pos, spot_tele):
|
|||
t4=int(t3/4)
|
||||
speed=t4*2
|
||||
r7=t3-t4*4
|
||||
gps=r7/2
|
||||
gps=int(r7/2)
|
||||
sats=r7%2
|
||||
|
||||
# print("T1-4,R7:",t1, t2, t3, t4, r7)
|
||||
|
|
@ -293,12 +292,12 @@ def decode_telemetry(spot_pos, spot_tele):
|
|||
loc=spot_pos_loc+subloc
|
||||
lat,lon = (maidenhead.toLoc(loc))
|
||||
|
||||
|
||||
print("%s Call: %6s Latlon: %10.5f %10.5f Loc: %6s Alt: %5d Temp: %6.2f Batt: %5.2f Speed: %3d GPS: %d Sats: %d" %
|
||||
pstr = ("Spot %s Call: %6s Latlon: %10.5f %10.5f Loc: %6s Alt: %5d Temp: %4.1f Batt: %5.2f Speed: %3d GPS: %1d Sats: %1d" %
|
||||
( spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ))
|
||||
logging.info(pstr)
|
||||
|
||||
telemetry = {'time':spot_pos_time, "call":spot_pos_call, "lat":lat, "lon":lon, "loc":loc, "alt": alt,
|
||||
"temp":temp, "batt":batt, "speed":speed, "gps":gps, "sats":sats }
|
||||
"temp":round(temp,1), "batt":round(batt,3), "speed":speed, "gps":gps, "sats":sats }
|
||||
|
||||
return telemetry
|
||||
|
||||
|
|
@ -354,10 +353,7 @@ def addsentdb(name, time_rec, sentence):
|
|||
return
|
||||
|
||||
|
||||
import logging
|
||||
from pprint import pformat
|
||||
|
||||
from subprocess import call
|
||||
|
||||
def send_tlm_to_habitat2(sentence, callsign):
|
||||
result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"])
|
||||
|
|
@ -366,7 +362,7 @@ def send_tlm_to_habitat2(sentence, callsign):
|
|||
def send_tlm_to_habitat(sentence, callsign, spot_time):
|
||||
input=sentence
|
||||
|
||||
print("Pushing data to habhub")
|
||||
logging.info("Pushing data to habhub")
|
||||
|
||||
|
||||
if not sentence.endswith('\n'):
|
||||
|
|
@ -394,35 +390,22 @@ def send_tlm_to_habitat(sentence, callsign, spot_time):
|
|||
},
|
||||
}
|
||||
|
||||
#x print(data)
|
||||
|
||||
h = httplib2.Http("")
|
||||
|
||||
resp, content = h.request(
|
||||
uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % sha256(sentence2).hexdigest(),
|
||||
uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(),
|
||||
method='PUT',
|
||||
headers={'Content-Type': 'application/json; charset=UTF-8'},
|
||||
body=json.dumps(data),
|
||||
)
|
||||
# logging.info("Response dictionary")
|
||||
# logging.info(pformat(resp))
|
||||
# logging.info("Response Content Body")
|
||||
# logging.info(pformat(content))
|
||||
|
||||
# print("Response dictionary")
|
||||
# print(pformat(resp))
|
||||
# print("Response Content Body")
|
||||
# print(pformat(content))
|
||||
|
||||
print(resp['status'])
|
||||
if resp['status'] == '201':
|
||||
print("OK 201", input)
|
||||
# sys.exit(0)
|
||||
elif resp['status'] == '403':
|
||||
print("Error 403 - already uploaded")
|
||||
|
||||
# time.sleep(1)
|
||||
|
||||
return
|
||||
|
||||
|
||||
|
|
@ -430,7 +413,6 @@ def send_tlm_to_habitat(sentence, callsign, spot_time):
|
|||
# Trim off older spots. Assuming oldest first!
|
||||
#
|
||||
def timetrim(spots, m):
|
||||
|
||||
if len(spots) == 0:
|
||||
return spots
|
||||
|
||||
|
|
@ -444,22 +426,11 @@ def timetrim(spots, m):
|
|||
spotc += 1
|
||||
# print(r[0], "vs ", time_last)
|
||||
if r[0] < time_last:
|
||||
# if splitspotc == 0:
|
||||
# print("split time found")
|
||||
splitspotc = spotc
|
||||
# break
|
||||
|
||||
l = len(spots)
|
||||
if splitspotc > 0:
|
||||
spots = spots[splitspotc:]
|
||||
# print("splitting")
|
||||
# else:
|
||||
# print("no splitting")
|
||||
|
||||
# print("Split pre",l,"splitc",spotc,"after:",len(spots),"time-cut", time_last, splitspotc)
|
||||
|
||||
# for r in spots:
|
||||
# print("splitlist:",r)
|
||||
|
||||
return spots
|
||||
|
||||
|
|
@ -475,23 +446,24 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
# print(row)
|
||||
if re.match('(^0|^Q).[0-9].*', row[1]):
|
||||
# print(', '.join(row))
|
||||
if re.match('10\..*', row[2]) or re.match('14\..*', row[2]):
|
||||
spots_tele.append(row)
|
||||
#if re.match('10\..*', row[2]) or re.match('14\..*', row[2]):
|
||||
spots_tele.append(row)
|
||||
|
||||
# 2018-05-03 13:06:00, QA5IQA, 7.040161, -8, JO53, 27, DH5RAE, JN68qv, 537
|
||||
# 0 1 2 3 4 5 6 7 8
|
||||
|
||||
for b in balloons:
|
||||
if len(spots) == 0:
|
||||
print("out of spots in balloonloop. returning")
|
||||
logging.info("out of spots in balloonloop. returning")
|
||||
return spots
|
||||
|
||||
balloon_name = b[0]
|
||||
balloon_call = b[1]
|
||||
balloon_mhz = b[2]
|
||||
balloon_channel = b[3]
|
||||
balloon_timeslot = b[4]
|
||||
|
||||
print("%s Name: %-6s Call: %6s MHz: %2d Channel: %2d " % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), balloon_name, balloon_call, balloon_mhz, balloon_channel), end='')
|
||||
logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot))
|
||||
|
||||
# Filter out telemetry for active channel
|
||||
if balloon_channel < 10:
|
||||
|
|
@ -499,81 +471,76 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
else:
|
||||
telem = [element for element in spots_tele if re.match('^Q.'+str(balloon_channel-10), element[1])]
|
||||
|
||||
# Filter out only selected band
|
||||
telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', element[2])]
|
||||
|
||||
print("Spots:", len(spots),"Telemetry:", len(telem))
|
||||
# if len(telem) > 0:
|
||||
# print("time at top-tele spot", telem[0])
|
||||
# time.sleep(2)
|
||||
# If timeslot is used. filter out correct slot
|
||||
if balloon_timeslot > 0:
|
||||
telem = [element for element in telem if balloon_timeslot == int(element[0].minute % 10 / 2)]
|
||||
|
||||
#if len(telem) > 0:
|
||||
# logging.info("time at top-tele spot: %s", str(telem[0]))
|
||||
|
||||
# Loop through all spots and try to find matching telemetrypacket
|
||||
spot_last = spots[0]
|
||||
spot_oldtime = datetime.datetime(1970, 1, 1, 1, 1)
|
||||
spot_oldtime = datetime.datetime(1970, 1, 1, 1, 1)
|
||||
bspots = []
|
||||
for row in spots:
|
||||
if balloon_call == row[1]:
|
||||
bspots.append(row)
|
||||
|
||||
logging.info("Spots: %d Telemetry: %d", len(bspots), len(telem))
|
||||
for row in bspots:
|
||||
# logging.info("B: %s",row)
|
||||
spot_call = row[1]
|
||||
|
||||
|
||||
if balloon_call == spot_call:
|
||||
# print(row)
|
||||
spot_time = row[0]
|
||||
# if posdata_cmp(row,spot_last):
|
||||
# print("Same payload")
|
||||
# sys.exit(0)
|
||||
# tdiff = spot_time
|
||||
|
||||
# if < timedelta(minutes=3) and tdiff > timedelta(minutes=0):
|
||||
|
||||
# Check if current spot is "much" older than latest telemetrypacket. If so, delete.
|
||||
# if len(telem) > 0:
|
||||
# print("time at top-tele spot", telem[0][0], row, telem[0], telem[0][0]-spot_time)
|
||||
# if (telem[0][0] - spot_time) > timedelta(minutes=10):
|
||||
# print("very big timediff detected", len(spots),len(spots_tele))
|
||||
# spots.remove(row)
|
||||
# time.sleep(0.1)
|
||||
|
||||
# Only check new uniq times
|
||||
# Only check new uniq times
|
||||
if spot_time != spot_oldtime:
|
||||
|
||||
# [datetime.datetime(2019, 8, 1, 0, 22), 'YO3ICT', '14.097148', -23, -1, 'BM73', 10, 'ND7M', 'DM16', 2564]
|
||||
pstr = "Call: %s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" % (row[1], row[0], row[2], row[5], row[6], row[7])
|
||||
logging.info(pstr)
|
||||
spot_oldtime = spot_time
|
||||
spot_fq = row[2]
|
||||
spot_loc = row[4]
|
||||
spot_power = row[5]
|
||||
spot_fq = row[2]
|
||||
spot_power = row[4]
|
||||
spot_loc = row[5]
|
||||
spot_reporter = row[6]
|
||||
spot_reporter_loc = row[7]
|
||||
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
|
||||
|
||||
|
||||
# Match positioningpacket with telemetrypackets
|
||||
b_telem = []
|
||||
for trow in telem:
|
||||
|
||||
# print("tdiff:",trow, spot_time)
|
||||
# print("tdiff:",trow, spot_time, type(spot_time))
|
||||
tdiff = trow[0] - spot_time
|
||||
# print(tdiff)
|
||||
# print(tdiff)
|
||||
|
||||
if tdiff > timedelta(minutes=10):
|
||||
print("too long interval, breaking", tdiff,trow)
|
||||
if tdiff > timedelta(minutes=8):
|
||||
logging.info("Too long interval, breaking %s", str(tdiff))
|
||||
break
|
||||
if tdiff > timedelta(minutes=0):
|
||||
b_telem.append(trow)
|
||||
|
||||
# time.sleep(0.1)
|
||||
# If suitable telemetry found, enter decoding!
|
||||
if len(b_telem) > 0:
|
||||
# print("Found suitable telemetry rows: ",len(b_telem))
|
||||
# print(b_telem[0])
|
||||
# for r in b_telem:
|
||||
# print("tele:", r, r[0]-spot_time)
|
||||
logging.info("Found suitable telemetry rows: %d",len(b_telem))
|
||||
|
||||
print("call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
|
||||
# print("call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
|
||||
#logging.info(pstr)
|
||||
#logging.info("T: %s", b_telem[0])
|
||||
telemetry = decode_telemetry(row, b_telem[0])
|
||||
if len(telemetry) > 0:
|
||||
# # Delete spot and telemetryspot
|
||||
# if row in spots:
|
||||
|
||||
# Delete spot and telemetryspot
|
||||
try:
|
||||
spots.remove(row)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
for rt in b_telem:
|
||||
print("Removing:", rt)
|
||||
pstr = "%s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" % (rt[1], rt[0], rt[2], rt[5], rt[6], rt[7])
|
||||
logging.info("Removing: %s", pstr)
|
||||
try:
|
||||
spots.remove(rt)
|
||||
except ValueError:
|
||||
|
|
@ -585,17 +552,14 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
pass
|
||||
|
||||
try:
|
||||
b_telem.remove(rt)
|
||||
telem.remove(rt)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
# print("Telemetry ok!")
|
||||
# print(telemetry)
|
||||
# logging.info(telemetry)
|
||||
|
||||
# seqnr = int(((int(telemetry['time'].strftime('%s'))) / 120) % 100000)
|
||||
seqnr = int(telemetry['time'].strftime('%s'))
|
||||
# print(seqnr)
|
||||
|
||||
# telemetry = [ spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ]
|
||||
telestr = "%s,%d,%s,%.5f,%.5f,%d,%d,%.2f,%.2f,%d,%d" % (
|
||||
|
|
@ -609,11 +573,11 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
checksum = checksum ^ ord(telestr[i])
|
||||
i+=1
|
||||
telestr = "$$" + telestr + "*" + '{:x}'.format(int(checksum))
|
||||
# print(telestr)
|
||||
logging.info("Telemetry: %s", telestr)
|
||||
|
||||
# Check if string has been uploaded before and if not then add and upload
|
||||
if not checkifsentdb(telestr):
|
||||
# print("Unsent spot", telestr)
|
||||
# print("Unsent spot", telestr)
|
||||
|
||||
if push_habhub:
|
||||
# Send telemetry to habhub
|
||||
|
|
@ -627,7 +591,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
sonde_data["lat"] = telemetry['lat']
|
||||
sonde_data["lon"] = telemetry['lon']
|
||||
sonde_data["alt"] = telemetry['alt']
|
||||
print("Pushing data to aprs.fi")
|
||||
logging.info("Pushing data to aprs.fi")
|
||||
push_balloon_to_aprs(sonde_data)
|
||||
|
||||
|
||||
|
|
@ -635,35 +599,10 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
|
|||
addsentdb(balloon_name, row[0], telestr)
|
||||
|
||||
else:
|
||||
print("Already sent spot. Doing nothing")
|
||||
logging.info("Already sent spot. Doing nothing")
|
||||
|
||||
return spots
|
||||
|
||||
|
||||
|
||||
# balloons = readballoonsdb()
|
||||
# spots = readgz(balloons, "wsprspots-2018-06.csv.gz")
|
||||
# spots.sort(reverse=True)
|
||||
#spots = readgz(balloons, "wsprspots-2018-05.csv.gz")
|
||||
# sys.exit(0)
|
||||
#dumpcsv(spots)
|
||||
# sys.exit(0)
|
||||
|
||||
# spots.sort(reverse=True)
|
||||
|
||||
# spots = readcsv()
|
||||
|
||||
# spots = spots + readnewspotsdb()
|
||||
|
||||
# Trim of spots older than x days
|
||||
# spots = timetrim(spots, 2)
|
||||
|
||||
|
||||
|
||||
#while True:
|
||||
# process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_apr)s
|
||||
|
||||
# print("Outer loop")
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
183
webscrape.py
183
webscrape.py
|
|
@ -4,7 +4,8 @@ from bs4 import BeautifulSoup
|
|||
import configparser
|
||||
import csv
|
||||
import datetime
|
||||
import re
|
||||
import getopt
|
||||
import os
|
||||
import requests
|
||||
import sqlite3
|
||||
import sys
|
||||
|
|
@ -13,23 +14,8 @@ import time
|
|||
from balloon import *
|
||||
from telemetry import *
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
config.read('balloon.ini')
|
||||
push_habhub = config['main']['push_habhub']
|
||||
push_aprs = config['main']['push_aprs']
|
||||
balloons = config['main']['balloons']
|
||||
|
||||
balloons = json.loads(config.get('main','balloons'))
|
||||
|
||||
print("Tracking these balloons:")
|
||||
for b in balloons:
|
||||
print(b)
|
||||
# print("Tracking these balloons:\n",type(balloons))
|
||||
|
||||
# sys.exit(0)
|
||||
|
||||
def getspots (nrspots):
|
||||
# print("Fetching...")
|
||||
# logging.info("Fetching...")
|
||||
wiki = "http://wsprnet.org/olddb?mode=html&band=all&limit=" + str(nrspots) + "&findcall=&findreporter=&sort=spotnum"
|
||||
try:
|
||||
page = requests.get(wiki)
|
||||
|
|
@ -37,8 +23,8 @@ def getspots (nrspots):
|
|||
print("ERROR",e)
|
||||
return []
|
||||
|
||||
# print(page.status)
|
||||
# print(page.data)
|
||||
# logging.info(page.status)
|
||||
# logging.info(page.data)
|
||||
|
||||
soup = BeautifulSoup(page.content, 'html.parser')
|
||||
|
||||
|
|
@ -122,7 +108,7 @@ def balloonfilter(spots,balloons):
|
|||
filtered.append(row)
|
||||
|
||||
# for r in filtered:
|
||||
# print("filtered out",r)
|
||||
# logging.info("filtered out",r)
|
||||
|
||||
return filtered
|
||||
|
||||
|
|
@ -134,35 +120,149 @@ def deduplicate(spotlist):
|
|||
rc_max = len(spotlist)-1
|
||||
if rc_max > 1:
|
||||
while rc < rc_max:
|
||||
# print("R:",rc, rc_max, len(spotlist))
|
||||
# print(spotlist[rc])
|
||||
# print(spotlist[rc+1])
|
||||
if (spotlist[rc][0] == spotlist[rc+1][0]) and (spotlist[rc][1] == spotlist[rc+1][1]):
|
||||
# print("Duplicate entry")
|
||||
# logging.info("Duplicate entry")
|
||||
del spotlist[rc]
|
||||
rc_max -= 1
|
||||
else:
|
||||
rc += 1
|
||||
|
||||
|
||||
# print("Deduplicate:",pre, len(spotlist))
|
||||
return spotlist
|
||||
|
||||
|
||||
# Setup logging
|
||||
|
||||
level = logging.INFO
|
||||
format = '%(asctime)s - %(message)s'
|
||||
handlers = [logging.FileHandler('logging.txt','a'), logging.StreamHandler()]
|
||||
logging.basicConfig(level = level, format = format, handlers = handlers, datefmt='%y-%m-%d %H:%M:%S')
|
||||
|
||||
#
|
||||
# Some options
|
||||
#
|
||||
|
||||
verbose = False
|
||||
archive_file = ''
|
||||
csv_file = ''
|
||||
conf_file = 'balloon.ini'
|
||||
dry_run = True
|
||||
|
||||
print("ARGV :", sys.argv[1:])
|
||||
|
||||
try:
|
||||
options, remainder = getopt.getopt(
|
||||
sys.argv[1:],
|
||||
'c:f:v',
|
||||
['archive=',
|
||||
'csv=',
|
||||
'conf=',
|
||||
])
|
||||
|
||||
except getopt.GetoptError as err:
|
||||
print('ERROR:', erxr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
logging.info("OPTIONS : %s", str(options))
|
||||
|
||||
for opt, arg in options:
|
||||
if opt in ('--archive'):
|
||||
archive_file = arg
|
||||
if opt in ('--csv'):
|
||||
csv_file = arg
|
||||
if opt in ('--conf'):
|
||||
conf_file = arg
|
||||
if opt in ('--dry_run'):
|
||||
dry_run = True
|
||||
elif opt in ('-v', '--verbose'):
|
||||
verbose = True
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
config.read(conf_file)
|
||||
push_habhub = config['main']['push_habhub']
|
||||
push_aprs = config['main']['push_aprs']
|
||||
# balloons = config['main']['balloons']
|
||||
|
||||
balloons = json.loads(config.get('main','balloons'))
|
||||
|
||||
logging.info("Tracking these balloons:")
|
||||
for b in balloons:
|
||||
logging.info("%s", str(b))
|
||||
|
||||
if dry_run:
|
||||
push_habhub = False
|
||||
push_aprs = False
|
||||
|
||||
spots = []
|
||||
|
||||
# Read active balloons from db
|
||||
# balloons = readballoonsdb()
|
||||
#
|
||||
# Load and process spots from archive-file - default append to csv
|
||||
#
|
||||
|
||||
# Spots to pull from wsprnet
|
||||
if archive_file:
|
||||
logging.info("Archive-mode")
|
||||
|
||||
# Read archivefile and filter out balloondata
|
||||
spots = readgz(balloons, archive_file)
|
||||
# logging.info(spots[0])
|
||||
spots.sort(reverse=False)
|
||||
|
||||
# Do a crude trimetrim
|
||||
# temp_spots = []
|
||||
#for s in spots:
|
||||
# if s[0] > datetime.datetime(2018, 5, 15, 0, 0) and s[0] > datetime.datetime(2018, 5, 15, 0, 0):
|
||||
# temp_spots.append(s)
|
||||
# spots = temp_spots
|
||||
|
||||
dumpcsv(spots)
|
||||
|
||||
if len(spots) > 1:
|
||||
logging.info("Spots: %s", str(len(spots)))
|
||||
spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
|
||||
else:
|
||||
logging.info("No spots!")
|
||||
|
||||
logging.info("Done")
|
||||
sys.exit(0)
|
||||
|
||||
#
|
||||
# Load and process spots from csv-file
|
||||
#
|
||||
|
||||
if csv_file:
|
||||
push_habhub = False
|
||||
push_aprs = False
|
||||
|
||||
spots = readcsv()
|
||||
|
||||
# Do a crude trimetrim
|
||||
# temp_spots = []
|
||||
# for s in spots:
|
||||
# if s[0] > datetime.datetime(2019, 12, 18, 11, 0) and s[0] < datetime.datetime(2019, 12, 18, 12, 30):
|
||||
# # print(s)
|
||||
# temp_spots.append(s)
|
||||
# spots = temp_spots
|
||||
|
||||
|
||||
if len(spots) > 1:
|
||||
logging.info("Spots: %s", str(len(spots)))
|
||||
spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
|
||||
else:
|
||||
logging.info("No spots!")
|
||||
|
||||
logging.info("Done")
|
||||
sys.exit(0)
|
||||
|
||||
# Spots to pullfrom wsprnet
|
||||
nrspots_pull= 2000
|
||||
spotcache = []
|
||||
|
||||
print("Preloading cache from wsprnet...")
|
||||
logging.info("Preloading cache from wsprnet...")
|
||||
spotcache = getspots(10000)
|
||||
print("Fspots1",len(spotcache))
|
||||
logging.info("Fspots1: %d",len(spotcache))
|
||||
spotcache = balloonfilter(spotcache ,balloons)
|
||||
print("Fspots2",len(spotcache))
|
||||
logging.info("Fspots2: %s", len(spotcache))
|
||||
|
||||
spots = spotcache
|
||||
cache_max = 10000
|
||||
|
|
@ -170,25 +270,18 @@ new_max = 0
|
|||
only_balloon=False
|
||||
sleeptime = 60
|
||||
|
||||
print("Entering pollingloop.")
|
||||
logging.info("Entering pollingloop.")
|
||||
while 1==1:
|
||||
tnow = datetime.datetime.now()
|
||||
|
||||
wwwspots = getspots(nrspots_pull)
|
||||
wwwspots = balloonfilter(wwwspots ,balloons)
|
||||
newspots = []
|
||||
#
|
||||
# wwwspots.reverse()
|
||||
|
||||
# for q in spotcache:
|
||||
# print("cache:",q)
|
||||
|
||||
# Sort in case some spots arrived out of order
|
||||
|
||||
spotcache.sort(reverse=False)
|
||||
spotcache = timetrim(spotcache,120)
|
||||
|
||||
|
||||
src_cc = 0
|
||||
|
||||
# Loop trough cache and check for new spots
|
||||
|
|
@ -203,7 +296,8 @@ while 1==1:
|
|||
break
|
||||
|
||||
if old == 0:
|
||||
print("New",row)
|
||||
# logging.info("New",str(row))
|
||||
logging.info("New spot: %s"<,row)
|
||||
|
||||
# Insert in beginning for cache
|
||||
spotcache.insert(0, row)
|
||||
|
|
@ -231,16 +325,16 @@ while 1==1:
|
|||
|
||||
|
||||
if len(spots) > 1:
|
||||
print("pre-tele:",len(spots))
|
||||
logging.info("pre-tele: %d",len(spots))
|
||||
spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
|
||||
# print("pro-tele:",len(spots))
|
||||
print("pro-tele:",len(spots))
|
||||
|
||||
if new_max < len(newspots):
|
||||
# and len(newspots) != nrspots_pull:
|
||||
new_max = len(newspots)
|
||||
|
||||
if len(newspots) == nrspots_pull:
|
||||
print("Hit max spots. Increasing set to fetch")
|
||||
logging.info("Hit max spots. Increasing set to fetch")
|
||||
nrspots_pull += 100
|
||||
|
||||
# print("%s Spots: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d Hitrate: %5.2f%%" %
|
||||
|
|
@ -251,9 +345,8 @@ while 1==1:
|
|||
|
||||
spotcache = spotcache[:cache_max]
|
||||
|
||||
|
||||
sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime
|
||||
# print("Sleep:", sleeping)
|
||||
# logging.info("Sleep:", sleeping)
|
||||
time.sleep(sleeping)
|
||||
|
||||
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue