From 589787e48af8832a6e8d983d0c7219824f27c798 Mon Sep 17 00:00:00 2001 From: sm3ulc Date: Sun, 22 Dec 2019 11:37:00 +0100 Subject: [PATCH] Big update with timeslots, loggging, command line options --- README.md | 59 +++++++++---- balloon.ini | 7 +- balloon.py | 55 ++++++------ sonde_to_aprs.py | 21 ++--- telemetry.py | 219 +++++++++++++++++------------------------------ webscrape.py | 183 +++++++++++++++++++++++++++++---------- 6 files changed, 302 insertions(+), 242 deletions(-) diff --git a/README.md b/README.md index ac85af7..a33b1d2 100644 --- a/README.md +++ b/README.md @@ -7,60 +7,83 @@ The software webscrapes data from wsprnet.org, filter out calls from the balloon * habhub tracker ( https://tracker.habhub.org/ ) * aprs-fi ( https://aprs.fi ). -There are existing functions to load/save flightdata from csv/wsprnet-archive-files. ( http://wsprnet.org/drupal/downloads ) +There are existing functions to load/save flightdata from csv/wsprnet-archive-files. +( http://wsprnet.org/drupal/downloads ) The protocol for the telemetry is described here: * https://qrp-labs.com/flights/s4.html + # Installation First clone the repo: - git clone https://github.com/sm3ulc/hab-wspr +
+git clone https://github.com/sm3ulc/hab-wspr
+
The package requires some extra modules that need to be installed via pip or similar - apt install python-httplib2 python-requests python3-bs4 - +
+apt install python-httplib2 python-requests python3-bs4
+
+ + +For windows users install anaconda with python 3. + +
+pip install httplib2
+pip install bs4
+
+ # Configuration Edit balloon.ini and add aprs-is user etc. Add balloons on the format: - [ habhub name, ham callsign for the balloon , band in mhz, channel ] +
+[ habhub name, ham callsign for the balloon , band in mhz, channel, timeslot ]
+
+ +timeslot = 0 to disable use of timeslots. 1-5, use correspondent slot 00, 02, 04 etc. + Uploads to APRS-IS is done by adding the SSID "-12" to the default balloon-callsign. -To run: +To run on linux: (with default config file balloon.ini) - python3 webscrape.py +
+python3 webscrape.py
+
 
 
 The scripts work with a database in sqlite. It can be used to do all kinds of output/export like checking the last sent spots:
 
-    sqlite3 wsprdb.db 'select * from sentspots order by time_sent desc limit 30'
+
+sqlite3 wsprdb.db 'select * from sentspots order by time_sent desc limit 30'
+

 
 # Testing
 
-Adjust your balloon.ini
+Adjust your balloon.ini or other configfile like test.ini.
 
 Goto http://wsprnet.org/drupal/downloads or
 
+
+wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz
+

 
-     wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz
-
-Extract data from archive and save filtered spots to spots.csv and then process. 
-
-      python3 webscrape.py --archive wsprspots-2019-12.csv.gz  --conf test.ini	 
+Extract data from archive and append filtered spots to spots.csv in and then process. 
 
+
+python3 webscrape.py --archive wsprspots-2019-12.csv.gz  --conf test.ini	 
+

 
 Read csv-file from spots.csv and process. 
 
-
python3 webscrape.py
-
---csv spots.csv
-
+
+python3 webscrape.py --csv spots.csv
 

 
 
diff --git a/balloon.ini b/balloon.ini
index f52a9b3..0b5659f 100644
--- a/balloon.ini
+++ b/balloon.ini
@@ -11,5 +11,8 @@ aprsUser = 'myaprsisuser'
 # APRS-IS passcode for your callsign.
 aprsPass = 'myaprsispass' 
 
-# [ habhub name, aprs-call, band in mhz, channel ] 
-balloons = [[ "BSY22","SA7XXX",14,11]]
+# [ habhub name, aprs-call, band in mhz, channel, timeslot ] 
+balloons = [[ "BSY22","SA7XXX",14,11,0 ],
+	 [ "B2", "SOMECALL",14,12,0 ]]
+
+
diff --git a/balloon.py b/balloon.py
index d110e71..e998358 100644
--- a/balloon.py
+++ b/balloon.py
@@ -1,6 +1,7 @@
 
 # from datetime import datetime,timedelta
 import datetime
+import logging
 import sqlite3
 import csv
 import sys
@@ -22,9 +23,9 @@ def balloonstodb(balloons):
             if not data:
                 con.commit()
         except sqlite3.Error as e:
-            print("Database error: %s" % e)
+                logging.info("Database error: %s",e)
         except Exception as e:
-            print("Exception in _query: %s" % e)
+                logging.info("Exception in _query:", e)
         finally:
             if con:
                 con.close()
@@ -60,6 +61,7 @@ def readballoonsdb():
 
 # Dumps all spots to csv-file
 def dumpcsv(spotlist):
+    logging.info("Writing spots to csv-file")
     with open('spots.csv', 'a', newline='') as csvfile:
         spotswriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
 
@@ -68,9 +70,10 @@ def dumpcsv(spotlist):
             spotswriter.writerow(row)
 
         csvfile.close()
+    return
 
 
-
+# Org-csv
 #  Date Call Frequency SNR Drift Grid dBm W reporter locator dist-km dist-mi 
 # 2018-05-21 19:04,F6HCO,10.140216,-11,1,J19bg,+33,1.995,SM0EPX/RX2,JO89si,1495,929
 
@@ -80,29 +83,33 @@ def dumpcsv(spotlist):
 # 2018-06-14 09:08,QA5IQB,5.288761,-26,0,JO22,+20,DL0HT,JO43jb,266
 # 0                1      2         3  4 5    6   7     8      9
 
+# Local-csv
+# 2018-05-01 02:14,QA5IQA,3.594159,-28,JO53,27,DC5AL-R,JO31lk,353
+# 2018-05-01 02:14,QA5IQA,3.594153,-21,JO53,27,DF2JP,JO31hh,380
+
+#  timestamp, tx_call, freq real, snr integer, drift integer, tx_loc, power , rx_call, rx_loc, distance
 
 def readcsv():
-    spots = []
-    with open('spots.csv', newline='') as csvfile:
-	    spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
+        spots = []
+        with open('spots.csv', newline='') as csvfile:
+                spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
 
-	    for row in spotsreader:
+                for row in spotsreader:
+                        # Time
+                        row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
+                        row[3] = int(row[3])
+                        row[4] = int(row[4])
+                        
+                        # Strip "+" from dB
+                        row[6] = int(row[6].replace('+',''))
+                        row[9] = int(row[9])
+                        
+                        # print(row)
+                        spots.append(row)
 
+        csvfile.close()
+        logging.info("Loaded spots: %s", len(spots))
+        #print("First",spots[1:][0])
+        #print("Last",spots[-1:][0])
 
-		    row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
-		    row[3] = int(row[3])
-		    row[4] = int(row[4])
-		    # Strip "+" from dB
-		    row[6] = int(row[6].replace('+',''))
-		    row[9] = int(row[9])
-
-#		    print(row)
-		    spots.append(row)
-
-    csvfile.close()
-#       print(spotsreader)
-    print("Loaded spots:", len(spots))
-    print("First",spots[1:][0])
-    print("Last",spots[-1:][0])
-
-    return spots
+        return spots
diff --git a/sonde_to_aprs.py b/sonde_to_aprs.py
index ffa8f7e..00dcacf 100755
--- a/sonde_to_aprs.py
+++ b/sonde_to_aprs.py
@@ -18,35 +18,30 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-
+import configparser
 import time, datetime, urllib3, sys
-# from pykml import parser
 from socket import *
 
 # APRS-IS login info
 serverHost = 'euro.aprs2.net' # Pick a local server if you like
 serverPort = 14580
 
-aprsUser = 'xxxx' # Replace with your own callsign. This should NOT have any SSID's on it (i.e. -9)
-aprsPass = '12345' # APRS-IS passcode for your callsign.
+config = configparser.ConfigParser()
+config.read('balloon.ini')
+aprsUser = config['main']['aprsUser']
+aprsPass = config['main']['aprsPass']
 
 # APRS packet Settings
-callsign = aprsUser # This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login. 
-callsign = "xxxx"
-
-# aprsUser # This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login. 
-
-update_rate = 30 # Update rate, in seconds.
+# This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login. 
+callsign = config['main']['aprsCallsign']
 
 # Get KML from SondeMonitor and parse into a Python dictionary
 def get_sonde():
 	sonde_data = {}
-
 	sonde_data["lat"] = str(sys.argv[2])
 	sonde_data["lon"] = str(sys.argv[3])
 	sonde_data["alt"] = "10000"
 	sonde_data["id"] = str(sys.argv[1])
-
 	return sonde_data
 
 # Push a Radiosonde data packet to APRS as an object.
@@ -82,7 +77,7 @@ def push_balloon_to_aprs(sonde_data):
 	alt = int(float(sonde_data["alt"])/0.3048)
 	
 	# Produce the APRS object string.
-	out_str = ";%s*111111z%s/%sO000/000/A=%06d Ballooon" % (object_name,lat_str,lon_str,alt)
+	out_str = ";%s*111111z%s/%sO000/000/A=%06d Balloon" % (object_name,lat_str,lon_str,alt)
 	print(out_str)
 
 	# Connect to an APRS-IS server, login, then push our object position in.
diff --git a/telemetry.py b/telemetry.py
index 9258edc..a831140 100644
--- a/telemetry.py
+++ b/telemetry.py
@@ -6,7 +6,9 @@ import csv
 import datetime
 from datetime import datetime,timedelta
 import gzip
+import hashlib
 import httplib2
+import logging
 import json
 import re
 import requests
@@ -14,6 +16,8 @@ import sqlite3
 import sys
 import time
 
+from pprint import pformat
+
 import maidenhead
 
 from balloon import *
@@ -45,14 +49,6 @@ def trim(spots):
         
     return spots
 
-#a = list(range(10))
-#a.reverse()
-#print(a)
-#a = trim(a)
-#print(a)
-#sys.exit(0)
-
-
 # Read new spots from database
 def readnewspotsdb():
     spots = []
@@ -112,8 +108,9 @@ def readnewspotsdb():
 
 
 def readgz(balloons, gzfile):
-    print("Reading gz:", gzfile)
-    
+    logging.info("Reading gz: %s", gzfile)
+
+    rows = 0
     spots = []
     calls = []
     for b in balloons:
@@ -123,29 +120,41 @@ def readgz(balloons, gzfile):
         spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')
 
         for row in spotsreader:
+            rows += 1
             # print(row)
             # Select correct fields in correct order
             row = [row[1], row[6], row[5], row[4], row[9], row[7], row[8],row[2],row[3],row[10]]
 
             # print(', '.join(row))
             for c in calls:
-#                print("Found", c, row)
                 if row[1] == c:
                     # Remove selfmade WSPR tranmissions
                     if len(row[5]) == 4:
                         row[0] = datetime.datetime.fromtimestamp(int(row[0]))
-#                        print("Found", c, row)
+                        row[3] = int(row[3])
+                        row[4] = int(row[4])
+
+                        # Strip "+" from dB
+                        row[6] = int(row[6].replace('+',''))
+                        row[9] = int(row[9])
+                        # print("Found", c, row)
                         spots.append(row)
 
             if re.match('(^0|^Q).[0-9].*', row[1]):
                 row[0] = datetime.datetime.fromtimestamp(int(row[0]))
- #               print("Found", row)
+                
+                row[3] = int(row[3])
+                row[4] = int(row[4])
+                        
+                # Strip "+" from dB
+                row[6] = int(row[6].replace('+',''))
+                row[9] = int(row[9])
+ 
                 spots.append(row)
+                # sys.exit(0)
 
-        print("Nr-calls+telem:", len(spots))
-
+        print("Total rows", rows, "Nr-calls+telem:", len(spots))
         csvfile.close()
-        print("Done reading")
 
     return spots
 
@@ -182,28 +191,20 @@ def decode_telemetry(spot_pos, spot_tele):
     c1 = spot_tele_call[1]
  #   print("C1=",c1)
     if c1.isalpha():
-        # print("c1=alpha")
         c1=ord(c1)-55
-        # print("new c1=",c1)    
     else:
         c1=ord(c1)-48
 
-    # print("new c1=",c1)  
-
     c2=ord(spot_tele_call[3])-65
     c3=ord(spot_tele_call[4])-65
     c4=ord(spot_tele_call[5])-65
 
-    # print("C:",c1,c2,c3,c4)
-
     # Convert locator to numbers
     l1=ord(spot_tele_loc[0])-65
     l2=ord(spot_tele_loc[1])-65
     l3=ord(spot_tele_loc[2])-48
     l4=ord(spot_tele_loc[3])-48
 
-    # print("L:",l1,l2,l3,l4)
-
     #
     # Convert power
     #
@@ -249,8 +250,6 @@ def decode_telemetry(spot_pos, spot_tele):
     lsub2=lsub2+65
     subloc=(chr(lsub1)+chr(lsub2)).lower()
 
-    # print("alt:",alt,"subloc:", lsub1, lsub2_tmp, lsub2,subloc)
-
     # Temperature
     # 40*42*2*2
     temp_1=int(sum2_tot/6720)
@@ -282,7 +281,7 @@ def decode_telemetry(spot_pos, spot_tele):
     t4=int(t3/4)
     speed=t4*2
     r7=t3-t4*4
-    gps=r7/2
+    gps=int(r7/2)
     sats=r7%2
 
     # print("T1-4,R7:",t1, t2, t3, t4, r7)
@@ -293,12 +292,12 @@ def decode_telemetry(spot_pos, spot_tele):
     loc=spot_pos_loc+subloc
     lat,lon = (maidenhead.toLoc(loc))
     
-
-    print("%s Call: %6s Latlon: %10.5f %10.5f Loc: %6s Alt: %5d Temp: %6.2f Batt: %5.2f Speed: %3d GPS: %d Sats: %d" %
+    pstr =    ("Spot %s Call: %6s Latlon: %10.5f %10.5f Loc: %6s Alt: %5d Temp: %4.1f Batt: %5.2f Speed: %3d GPS: %1d Sats: %1d" %
           (  spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ))
+    logging.info(pstr)
 
     telemetry = {'time':spot_pos_time, "call":spot_pos_call, "lat":lat, "lon":lon, "loc":loc, "alt": alt,
-                 "temp":temp, "batt":batt, "speed":speed, "gps":gps, "sats":sats }
+                 "temp":round(temp,1), "batt":round(batt,3), "speed":speed, "gps":gps, "sats":sats }
 
     return telemetry
 
@@ -354,10 +353,7 @@ def addsentdb(name, time_rec, sentence):
     return
 
 
-import logging
-from pprint import pformat
 
-from subprocess import call
 
 def send_tlm_to_habitat2(sentence, callsign):
     result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"])
@@ -366,7 +362,7 @@ def send_tlm_to_habitat2(sentence, callsign):
 def send_tlm_to_habitat(sentence, callsign, spot_time):
     input=sentence
 
-    print("Pushing data to habhub")
+    logging.info("Pushing data to habhub")
 
 
     if not sentence.endswith('\n'):
@@ -394,35 +390,22 @@ def send_tlm_to_habitat(sentence, callsign, spot_time):
             },
         }
 
-#x    print(data)
 
     h = httplib2.Http("")
 
     resp, content = h.request(
-        uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % sha256(sentence2).hexdigest(),
+        uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(),
         method='PUT',
         headers={'Content-Type': 'application/json; charset=UTF-8'},
         body=json.dumps(data),
     )
-#    logging.info("Response dictionary")
-#    logging.info(pformat(resp))
-#    logging.info("Response Content Body")
-#    logging.info(pformat(content))
-
-#    print("Response dictionary")
-#    print(pformat(resp))
-#    print("Response Content Body")
-#    print(pformat(content))
 
     print(resp['status'])
     if resp['status'] == '201':
         print("OK 201", input)
-#        sys.exit(0)
     elif resp['status'] == '403':
         print("Error 403 - already uploaded")
 
-#    time.sleep(1)
-
     return
 
 
@@ -430,7 +413,6 @@ def send_tlm_to_habitat(sentence, callsign, spot_time):
 # Trim off older spots. Assuming oldest first!
 #
 def timetrim(spots, m):
-
     if len(spots) == 0:
         return spots
 
@@ -444,22 +426,11 @@ def timetrim(spots, m):
         spotc += 1
 #        print(r[0], "vs ", time_last)
         if r[0] < time_last:
-            #                if splitspotc == 0:
-#            print("split time found")
             splitspotc = spotc
-#            break
 
     l = len(spots)
     if splitspotc > 0:
         spots = spots[splitspotc:]
- #       print("splitting")
-#    else:
- #       print("no splitting")
-
-#    print("Split pre",l,"splitc",spotc,"after:",len(spots),"time-cut", time_last, splitspotc)
-
-#    for r in spots:
-#        print("splitlist:",r)
         
     return spots
 
@@ -475,23 +446,24 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
         # print(row)
         if re.match('(^0|^Q).[0-9].*', row[1]):
             #         print(', '.join(row))
-            if re.match('10\..*', row[2]) or re.match('14\..*', row[2]):
-                spots_tele.append(row)
+            #if re.match('10\..*', row[2]) or re.match('14\..*', row[2]):
+            spots_tele.append(row)
 
     # 2018-05-03 13:06:00, QA5IQA, 7.040161, -8, JO53, 27, DH5RAE, JN68qv, 537
     # 0                    1       2         3   4     5   6       7       8 
 
     for b in balloons:
         if len(spots) == 0:
-            print("out of spots in balloonloop. returning")
+            logging.info("out of spots in balloonloop. returning")
             return spots
             
         balloon_name = b[0]
         balloon_call = b[1]
         balloon_mhz = b[2]
         balloon_channel = b[3]
+        balloon_timeslot = b[4]
 
-        print("%s Name: %-6s Call: %6s MHz: %2d Channel: %2d " % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), balloon_name, balloon_call, balloon_mhz, balloon_channel), end='')
+        logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot))
 
         # Filter out telemetry for active channel
         if balloon_channel < 10:
@@ -499,81 +471,76 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
         else:
             telem = [element for element in spots_tele if re.match('^Q.'+str(balloon_channel-10), element[1])]
 
+        # Filter out only selected band
+        telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', element[2])]
 
-        print("Spots:", len(spots),"Telemetry:", len(telem))
-#        if len(telem) > 0:
-#            print("time at top-tele spot", telem[0])
-#        time.sleep(2)
+        # If timeslot is used. filter out correct slot
+        if balloon_timeslot > 0:
+            telem = [element for element in telem if balloon_timeslot == int(element[0].minute % 10 / 2)]
+                                
+        #if len(telem) > 0:
+            # logging.info("time at top-tele spot: %s", str(telem[0]))
 
         # Loop through all spots and try to find matching telemetrypacket
         spot_last = spots[0]
-        spot_oldtime = datetime.datetime(1970, 1, 1, 1, 1) 
+        spot_oldtime = datetime.datetime(1970, 1, 1, 1, 1)
+        bspots = []
         for row in spots:
+            if balloon_call == row[1]:
+                bspots.append(row)
+                
+        logging.info("Spots: %d  Telemetry: %d", len(bspots), len(telem))
+        for row in bspots:
+            # logging.info("B: %s",row)
             spot_call = row[1]
-
+            
             if balloon_call == spot_call:
-#                print(row)
                 spot_time = row[0]
-    #            if posdata_cmp(row,spot_last):
-    #                print("Same payload")
-    #                sys.exit(0)
-    #            tdiff = spot_time
 
-    #                if  < timedelta(minutes=3) and tdiff > timedelta(minutes=0):
-
-                # Check if current spot is "much" older than latest telemetrypacket. If so, delete.
-#                if len(telem) > 0:
-    #                print("time at top-tele spot", telem[0][0], row, telem[0], telem[0][0]-spot_time)
-#                    if (telem[0][0] - spot_time) > timedelta(minutes=10):
-#                        print("very big timediff detected", len(spots),len(spots_tele))
-#                        spots.remove(row)
-#                        time.sleep(0.1)
-
-                 # Only check new uniq times
+                # Only check new uniq times
                 if spot_time != spot_oldtime:
+
+                    # [datetime.datetime(2019, 8, 1, 0, 22), 'YO3ICT', '14.097148', -23, -1, 'BM73', 10, 'ND7M', 'DM16', 2564]
+                    pstr = "Call: %s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" %  (row[1], row[0], row[2], row[5], row[6], row[7])
+                    logging.info(pstr)
                     spot_oldtime = spot_time
-                    spot_fq = row[2]        
-                    spot_loc = row[4]
-                    spot_power = row[5]
+                    spot_fq = row[2]
+                    spot_power = row[4]
+                    spot_loc = row[5]
                     spot_reporter = row[6]        
                     spot_reporter_loc = row[7]
-                    print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
-
+                    
                 # Match positioningpacket with telemetrypackets
                     b_telem = []
                     for trow in telem:
-
-#                        print("tdiff:",trow, spot_time)
+                        # print("tdiff:",trow, spot_time, type(spot_time))
                         tdiff = trow[0] - spot_time
-    #                    print(tdiff)
+                        #                    print(tdiff)
 
-                        if tdiff > timedelta(minutes=10):
-                            print("too long interval, breaking", tdiff,trow)
+                        if tdiff > timedelta(minutes=8):
+                            logging.info("Too long interval, breaking %s", str(tdiff))
                             break
                         if tdiff > timedelta(minutes=0):
                             b_telem.append(trow)
 
-    #                    time.sleep(0.1)    
                     # If suitable telemetry found, enter decoding!
                     if len(b_telem) > 0:
-#                        print("Found suitable telemetry rows: ",len(b_telem))
-    #                    print(b_telem[0])
-    #                    for r in b_telem:
-    #                        print("tele:", r, r[0]-spot_time)
+                        logging.info("Found suitable telemetry rows: %d",len(b_telem))
 
-                        print("call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
+                        # print("call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
+                        #logging.info(pstr)
+                        #logging.info("T: %s", b_telem[0])
                         telemetry = decode_telemetry(row, b_telem[0])
                         if len(telemetry) > 0:
-                            #                            # Delete spot and telemetryspot
-                            #                            if row in spots:
-
+                            # Delete spot and telemetryspot
                             try:
                                 spots.remove(row)
                             except ValueError:
                                 pass
 
                             for rt in b_telem:
-                                print("Removing:", rt)
+                                pstr = "%s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" %  (rt[1], rt[0], rt[2], rt[5], rt[6], rt[7]) 
+                                logging.info("Removing: %s", pstr)
                                 try:
                                     spots.remove(rt)
                                 except ValueError:
@@ -585,17 +552,14 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
                                     pass
 
                                 try:
-                                    b_telem.remove(rt)
+                                    telem.remove(rt)
                                 except ValueError:
                                     pass
 
-
-    #                        print("Telemetry ok!")
-    #                        print(telemetry)
+                            # logging.info(telemetry)
 
                             # seqnr = int(((int(telemetry['time'].strftime('%s'))) / 120) % 100000)
                             seqnr = int(telemetry['time'].strftime('%s'))
-#                            print(seqnr)
 
                             # telemetry = [ spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ]
                             telestr = "%s,%d,%s,%.5f,%.5f,%d,%d,%.2f,%.2f,%d,%d" % (  
@@ -609,11 +573,11 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
                                 checksum = checksum ^ ord(telestr[i])
                                 i+=1
                             telestr = "$$" + telestr + "*" + '{:x}'.format(int(checksum))
-    #                        print(telestr)
+                            logging.info("Telemetry: %s", telestr)
 
                             # Check if string has been uploaded before and if not then add and upload
                             if not checkifsentdb(telestr):
-#                                 print("Unsent spot", telestr)
+                                # print("Unsent spot", telestr)
 
                                 if push_habhub:
                                     # Send telemetry to habhub
@@ -627,7 +591,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
                                     sonde_data["lat"] = telemetry['lat']
                                     sonde_data["lon"] = telemetry['lon']
                                     sonde_data["alt"] = telemetry['alt']
-                                    print("Pushing data to aprs.fi")
+                                    logging.info("Pushing data to aprs.fi")
                                     push_balloon_to_aprs(sonde_data)
 
 
@@ -635,35 +599,10 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs):
                                 addsentdb(balloon_name, row[0], telestr)
 
                             else:
-                                print("Already sent spot. Doing nothing")
+                                logging.info("Already sent spot. Doing nothing")
 
     return spots
 
 
 
-# balloons = readballoonsdb()
-# spots = readgz(balloons, "wsprspots-2018-06.csv.gz")
-# spots.sort(reverse=True)
-#spots = readgz(balloons, "wsprspots-2018-05.csv.gz")
-# sys.exit(0)
-#dumpcsv(spots)
-# sys.exit(0)
-
-# spots.sort(reverse=True)
-
-# spots = readcsv()
-
-# spots = spots + readnewspotsdb()
-
-# Trim of spots older than x days
-# spots = timetrim(spots, 2)
-
-
-
-#while True:
-#    process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_apr)s
-
-#    print("Outer loop")
-
-
         
diff --git a/webscrape.py b/webscrape.py
index c1dc0a8..e29560b 100755
--- a/webscrape.py
+++ b/webscrape.py
@@ -4,7 +4,8 @@ from bs4 import BeautifulSoup
 import configparser
 import csv
 import datetime
-import re
+import getopt
+import os
 import requests
 import sqlite3
 import sys
@@ -13,23 +14,8 @@ import time
 from balloon import *
 from telemetry import *
 
-config = configparser.ConfigParser()
-config.read('balloon.ini')
-push_habhub = config['main']['push_habhub']
-push_aprs = config['main']['push_aprs']
-balloons = config['main']['balloons']
-
-balloons = json.loads(config.get('main','balloons'))
-
-print("Tracking these balloons:")
-for b in balloons:
-      print(b)
-# print("Tracking these balloons:\n",type(balloons))
-
-# sys.exit(0)
-
 def getspots (nrspots):
-#    print("Fetching...")
+#    logging.info("Fetching...")
     wiki = "http://wsprnet.org/olddb?mode=html&band=all&limit=" + str(nrspots) + "&findcall=&findreporter=&sort=spotnum"
     try:
         page = requests.get(wiki)
@@ -37,8 +23,8 @@ def getspots (nrspots):
         print("ERROR",e)
         return []
 
-#    print(page.status)
-#    print(page.data)
+#    logging.info(page.status)
+#    logging.info(page.data)
 
     soup = BeautifulSoup(page.content, 'html.parser')
 
@@ -122,7 +108,7 @@ def balloonfilter(spots,balloons):
                 filtered.append(row)
 
 #    for r in filtered:
-#        print("filtered out",r)
+#        logging.info("filtered out",r)
 
     return filtered
 
@@ -134,35 +120,149 @@ def deduplicate(spotlist):
     rc_max = len(spotlist)-1
     if rc_max > 1:
         while rc < rc_max:
-#            print("R:",rc, rc_max, len(spotlist))
-#            print(spotlist[rc])
-#            print(spotlist[rc+1])
             if (spotlist[rc][0] == spotlist[rc+1][0]) and (spotlist[rc][1] == spotlist[rc+1][1]):
-#                print("Duplicate entry")
+#                logging.info("Duplicate entry")
                 del spotlist[rc]
                 rc_max -= 1
             else:
                 rc += 1
 
-
 #    print("Deduplicate:",pre, len(spotlist))
     return spotlist
 
 
+# Setup logging
+
+level    = logging.INFO
+format   = '%(asctime)s - %(message)s'
+handlers = [logging.FileHandler('logging.txt','a'), logging.StreamHandler()]
+logging.basicConfig(level = level, format = format, handlers = handlers, datefmt='%y-%m-%d %H:%M:%S')
+
+#
+# Some options
+# 
+
+verbose = False
+archive_file = ''
+csv_file = ''
+conf_file = 'balloon.ini'
+dry_run = True
+
+print("ARGV      :", sys.argv[1:])
+
+try:
+      options, remainder = getopt.getopt(
+            sys.argv[1:],
+            'c:f:v',
+                ['archive=',
+                 'csv=',
+                 'conf=',
+                ])
+
+except getopt.GetoptError as err:
+    print('ERROR:', erxr)
+    sys.exit(1)
+
+
+logging.info("OPTIONS   : %s", str(options))
+      
+for opt, arg in options:
+    if opt in ('--archive'):
+        archive_file = arg
+    if opt in ('--csv'):
+        csv_file = arg
+    if opt in ('--conf'):
+        conf_file = arg
+    if opt in ('--dry_run'):
+        dry_run = True
+    elif opt in ('-v', '--verbose'):
+        verbose = True
+
+config = configparser.ConfigParser()
+config.read(conf_file)
+push_habhub = config['main']['push_habhub']
+push_aprs = config['main']['push_aprs']
+# balloons = config['main']['balloons']
+
+balloons = json.loads(config.get('main','balloons'))
+            
+logging.info("Tracking these balloons:")
+for b in balloons:
+      logging.info("%s", str(b))
+
+if dry_run:
+      push_habhub = False
+      push_aprs = False
+      
 spots = []
 
-# Read active balloons from db
-# balloons = readballoonsdb()
+#
+# Load and process spots from archive-file - default append to csv
+#
 
-# Spots to pull from wsprnet
+if archive_file:
+      logging.info("Archive-mode")
+
+      # Read archivefile and filter out balloondata
+      spots = readgz(balloons, archive_file)
+      # logging.info(spots[0])
+      spots.sort(reverse=False)
+
+      # Do a crude trimetrim 
+      # temp_spots = []
+      #for s in spots:
+      #      if s[0] > datetime.datetime(2018, 5, 15, 0, 0) and s[0] > datetime.datetime(2018, 5, 15, 0, 0):
+      #            temp_spots.append(s)
+      # spots = temp_spots
+      
+      dumpcsv(spots)
+
+      if len(spots) > 1:
+            logging.info("Spots: %s", str(len(spots)))
+            spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
+      else:
+            logging.info("No spots!")
+            
+      logging.info("Done")
+      sys.exit(0)
+
+#
+# Load and process spots from csv-file
+#
+      
+if csv_file:
+      push_habhub = False
+      push_aprs = False
+
+      spots = readcsv()
+
+      # Do a crude trimetrim 
+      # temp_spots = []
+      # for s in spots:
+      #     if s[0] > datetime.datetime(2019, 12, 18, 11, 0) and s[0] < datetime.datetime(2019, 12, 18, 12, 30):
+      #         # print(s)
+      #         temp_spots.append(s)
+      # spots = temp_spots
+
+      
+      if len(spots) > 1:
+            logging.info("Spots: %s", str(len(spots)))
+            spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
+      else:
+            logging.info("No spots!")
+
+      logging.info("Done")
+      sys.exit(0)
+            
+# Spots to pullfrom wsprnet
 nrspots_pull= 2000
 spotcache = []
 
-print("Preloading cache from wsprnet...")
+logging.info("Preloading cache from wsprnet...")
 spotcache = getspots(10000)
-print("Fspots1",len(spotcache))
+logging.info("Fspots1: %d",len(spotcache))
 spotcache = balloonfilter(spotcache ,balloons)
-print("Fspots2",len(spotcache))
+logging.info("Fspots2: %s", len(spotcache))
 
 spots = spotcache
 cache_max = 10000
@@ -170,25 +270,18 @@ new_max = 0
 only_balloon=False
 sleeptime = 60
 
-print("Entering pollingloop.")
+logging.info("Entering pollingloop.")
 while 1==1:
     tnow = datetime.datetime.now() 
 
     wwwspots = getspots(nrspots_pull)
     wwwspots = balloonfilter(wwwspots ,balloons)
     newspots = [] 
-    # 
-    # wwwspots.reverse()
-
-#    for q in spotcache:
-#        print("cache:",q)
 
     # Sort in case some spots arrived out of order
-
     spotcache.sort(reverse=False)   
     spotcache = timetrim(spotcache,120)
 
-
     src_cc = 0 
 
     # Loop trough cache and check for new spots
@@ -203,7 +296,8 @@ while 1==1:
                 break
 
         if old == 0:
-            print("New",row)
+            # logging.info("New",str(row))
+            logging.info("New spot: %s"<,row)
             
             # Insert in beginning for cache
             spotcache.insert(0, row)
@@ -231,16 +325,16 @@ while 1==1:
 
 
     if len(spots) > 1:
-        print("pre-tele:",len(spots))
+        logging.info("pre-tele: %d",len(spots))
         spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
-        # print("pro-tele:",len(spots))
+        print("pro-tele:",len(spots))
 
     if new_max < len(newspots):
 #  and len(newspots) != nrspots_pull:
         new_max = len(newspots)
 
     if len(newspots) == nrspots_pull:
-        print("Hit max spots. Increasing set to fetch")
+        logging.info("Hit max spots. Increasing set to fetch")
         nrspots_pull += 100
 
 #    print("%s Spots: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d Hitrate: %5.2f%%" % 
@@ -251,9 +345,8 @@ while 1==1:
 
     spotcache = spotcache[:cache_max]
 
-
     sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime
-#     print("Sleep:", sleeping)
+#     logging.info("Sleep:", sleeping)
     time.sleep(sleeping)