kopia lustrzana https://github.com/sm3ulc/hab-wspr
new wsprnet format
rodzic
2a7ae4463b
commit
39d43a181a
25
webscrape.py
25
webscrape.py
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import configparser
|
import configparser
|
||||||
|
@ -30,7 +30,7 @@ def getspots (nrspots):
|
||||||
|
|
||||||
data = []
|
data = []
|
||||||
table = soup.find_all('table')[2]
|
table = soup.find_all('table')[2]
|
||||||
# print("TABLE:",table)
|
#print("TABLE:",table)
|
||||||
|
|
||||||
rows = table.findAll('tr')
|
rows = table.findAll('tr')
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
@ -43,9 +43,11 @@ def getspots (nrspots):
|
||||||
|
|
||||||
# Strip redundant columns Watt & miles and translate/filter data
|
# Strip redundant columns Watt & miles and translate/filter data
|
||||||
for row in newspots:
|
for row in newspots:
|
||||||
|
#print(row)
|
||||||
row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
|
row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
|
||||||
row[6] = int(row[6].replace('+',''))
|
row[6] = int(row[6].replace('+',''))
|
||||||
|
|
||||||
|
del row[12]
|
||||||
del row[11]
|
del row[11]
|
||||||
del row[7]
|
del row[7]
|
||||||
|
|
||||||
|
@ -144,7 +146,7 @@ csv_file = ''
|
||||||
conf_file = 'balloon.ini'
|
conf_file = 'balloon.ini'
|
||||||
dry_run = False
|
dry_run = False
|
||||||
|
|
||||||
print("ARGV :", sys.argv[1:])
|
#print("ARGV :", sys.argv[1:])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
options, remainder = getopt.getopt(
|
options, remainder = getopt.getopt(
|
||||||
|
@ -160,7 +162,7 @@ except getopt.GetoptError as err:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
logging.info("OPTIONS : %s", str(options))
|
#logging.info("OPTIONS : %s", str(options))
|
||||||
|
|
||||||
for opt, arg in options:
|
for opt, arg in options:
|
||||||
if opt in ('--archive'):
|
if opt in ('--archive'):
|
||||||
|
@ -324,7 +326,7 @@ while 1==1:
|
||||||
if len(spots) > 1:
|
if len(spots) > 1:
|
||||||
logging.info("pre-tele: %d",len(spots))
|
logging.info("pre-tele: %d",len(spots))
|
||||||
spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
|
spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
|
||||||
print("pro-tele:",len(spots))
|
logging.info("pro-tele: %s", str(len(spots)))
|
||||||
|
|
||||||
if new_max < len(newspots):
|
if new_max < len(newspots):
|
||||||
# and len(newspots) != nrspots_pull:
|
# and len(newspots) != nrspots_pull:
|
||||||
|
@ -334,12 +336,15 @@ while 1==1:
|
||||||
logging.info("Hit max spots. Increasing set to fetch")
|
logging.info("Hit max spots. Increasing set to fetch")
|
||||||
nrspots_pull += 100
|
nrspots_pull += 100
|
||||||
|
|
||||||
# print("%s Spots: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d Hitrate: %5.2f%%" %
|
# print("%s Spots: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d Hitrate: %5.2f%%" %
|
||||||
# (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc, 100-(src_cc / (len(spotcache)*nrspots_pull))*100))
|
# (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc, 100-(src_cc / (len(spotcache)*nrspots_pull))*100))
|
||||||
|
|
||||||
print("%s Spots: %5d Cache: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d" %
|
|
||||||
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spots), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc))
|
|
||||||
|
|
||||||
|
# print("%s Spots: %5d Cache: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d" %
|
||||||
|
# (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spots), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc))
|
||||||
|
printstr = ("Spots: %5d Cache: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d" %
|
||||||
|
(len(spots), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc))
|
||||||
|
logging.info(printstr)
|
||||||
|
|
||||||
spotcache = spotcache[:cache_max]
|
spotcache = spotcache[:cache_max]
|
||||||
|
|
||||||
sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime
|
sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime
|
||||||
|
|
Ładowanie…
Reference in New Issue