hab-wspr/telemetry.py

#!/usr/bin/env python3

from base64 import b64encode
import configparser
import csv
import datetime
from datetime import datetime,timedelta
import gzip
import hashlib
import httplib2
import logging
import json
import re
import requests
import sqlite3
import sys
import time

from pprint import pformat

import maidenhead

from balloon import *
from sonde_to_aprs import * 
from sonde_to_html import *

# Power to decixmal conversion table 
pow2dec = {0:0,3:1,7:2,10:3,13:4,17:5,20:6,23:7,27:8,30:9,33:10,37:11,40:12,43:13,47:14,50:15,53:16,57:17,60:18}

config = configparser.ConfigParser()
config.read('balloon.ini')
habhub_callsign = config['main']['habhub_callsign']

def trim(spots):
    # Clean out old spots
    if len(spots) > 0:
        print(spots[-1:])
        time_last = 8
        spotc = 0
        splitspotc = 0
        for r in spots:
            spotc += 1
            if time_last < r:
                if splitspotc == 0:
                    splitspotc = spotc

        l = len(spots)
        spots = spots[splitspotc:]
        print("Split pre",l,"splitc",spotc,"after:",len(spots))
        
    return spots

# Read new spots from database
def readnewspotsdb():
    spots = []
    con = None
    data = None
    
    try:
        con = sqlite3.connect('wsprdb.db')
        cur = con.cursor()
        cur.execute('select * from newspots')
        data = cur.fetchall()

        for row in data:
#            print(row)'
            row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
            spots.append(list(row))
#            sys.exit(0)

        if not data:
            con.commit()
    except sqlite3.Error as e:
        print("Database error: %s" % e)
    except Exception as e:
        print("Exception in _query: %s" % e)
    finally:
        if con:
            con.close()
    print("Loaded spots:", len(spots))
    return spots


#
# Specs of the wspr-db format
#
# 1  Spot ID - A unique integer. Used as primary key in the database table. Not all spot numbers exist, and the files may not be in spot number order
# 2  Timestamp - The time of the spot in unix time() format (seconds since 1970-01-01 00:00 UTC).
# 3  Reporter - The station reporting the spot. Maximum of 10 characters.
# 4  Reporter's Grid - Maidenhead grid locator of the reporting station, in 4- or 6-character format.
# 5  SNR - Signal to noise ratio in dB as reported by the receiving software.
# 6  Frequency - Frequency of the received signal in MHz
# 7  Call Sign - Call sign of the transmitting station. 
# 8  Grid - Maidenhead grid locator of transmitting station, in 4- or 6-character format.
# 9  Power - Power, as reported by transmitting station in the transmission. 
# 10 Drift - The measured drift of the transmitted signal as seen by the receiver, in Hz/minute. 
# 11 Distance - Approximate distance between transmitter and receiver
# 12 Azimuth - Approximate direction, in degrees, from transmitting station to receiving station.
# 13 Band - Band of operation, computed from frequency as an index for faster retrieval. 
# 14 Version - Version string of the WSPR software in use by the receiving station. 
# 15 Code - Archives generated after 22 Dec 2010 have an additional integer Code field

# 1130358407,1522540800,DC0DX/MW2,JO31lk,-28,0.137553,2E0ILY,IO82qv,23,0,673,100,-1,,0

# 2018-05-28 05:50,OM1AI,7.040137,-15,0,JN88,+23,DA5UDI,JO30qj,724
# timestamp, tx_call , freq, snr , drift , tx_loc , power , rx_call, rx_loc, distance 
# 0          1         2     3     4       5        6       7        8       9



def readgz(balloons, gzfile):
    logging.info("Reading gz: %s", gzfile)

    rows = 0
    spots = []
    calls = []
    for b in balloons:
        calls.append(b[1])

    with gzip.open(gzfile, "rt") as csvfile:
        spotsreader = csv.reader(csvfile, delimiter=',', quotechar='|')

        for row in spotsreader:
            rows += 1
            # print(row)
            # Select correct fields in correct order
            row = [row[1], row[6], row[5], row[4], row[9], row[7], row[8],row[2],row[3],row[10]]

            # print(', '.join(row))
            for c in calls:
                if row[1] == c:
                    # Remove selfmade WSPR tranmissions
                    if len(row[5]) == 4:
                        row[0] = datetime.datetime.fromtimestamp(int(row[0]))
                        row[3] = int(row[3])
                        row[4] = int(row[4])

                        # Strip "+" from dB
                        row[6] = int(row[6].replace('+',''))
                        row[9] = int(row[9])
                        # print("Found", c, row)
                        spots.append(row)

            if re.match('(^0|^1|^Q).[0-9].*', row[1]):
                row[0] = datetime.datetime.fromtimestamp(int(row[0]))
                
                row[3] = int(row[3])
                row[4] = int(row[4])
                        
                # Strip "+" from dB
                row[6] = int(row[6].replace('+',''))
                row[9] = int(row[9])
 
                spots.append(row)
                # sys.exit(0)

        print("Total rows", rows, "Nr-calls+telem:", len(spots))
        csvfile.close()

    return spots


def posdata_cmp(spot1, spot2):
    if [spot1[1],spot1[5],spot1[6]] == [spot2[1],spot2[5],spot2[6]]:
        print("lika")
        return True
    else:
        print("olika")
        return False

#  ['2018-05-15 18:14', 'SA6BSS', '14.097165', '-21', '0', 'AN84', '+13', '0.020', 'AI6VN/KH6', 'BL10rx', '2681', '1666']
#  [datetime.datetime(2018, 5, 15, 18, 16), 'Q11DCN', '14.097184', '-25', '1', 'FB18', '+30', '1.000', 'JH1HRJ', 'PM95pi', '15479', '9618']

# [datetime.datetime(2018, 6, 1, 5, 44), 'SA6BSS', '14.097174', -19, 1, 'MO15', 13, 'LA9JO', 'JP99gb', 2659] 
#  [datetime.datetime(2018, 6, 1, 5, 46), 'QK1TKY', '14.097174', -22, 0, 'FB17', 50, 'LA9JO', 'JP99gb', 17160]

# 2018-05-03 13:06:00, QA5IQA, 7.040161, -8, JO53, 27, DH5RAE, JN68qv, 537
# 0                    1       2         3   4     5   6       7       8 

def decode_telemetry(spot_pos, spot_tele):
#    print("Decoding!\n",spot_pos,"\n",spot_tele)

    spot_pos_time = spot_pos[0]
    spot_pos_call = spot_pos[1]
    spot_pos_loc = spot_pos[5]
    spot_pos_power = spot_pos[6]
    spot_tele_call = spot_tele[1]
    spot_tele_loc = spot_tele[5]
    spot_tele_power = spot_tele[6]
    
    # Convert call to numbers
    c1 = spot_tele_call[1]
 #   print("C1=",c1)
    if c1.isalpha():
        c1=ord(c1)-55
    else:
        c1=ord(c1)-48

    c2=ord(spot_tele_call[3])-65
    c3=ord(spot_tele_call[4])-65
    c4=ord(spot_tele_call[5])-65

    # Convert locator to numbers
    l1=ord(spot_tele_loc[0])-65
    l2=ord(spot_tele_loc[1])-65
    l3=ord(spot_tele_loc[2])-48
    l4=ord(spot_tele_loc[3])-48

    #
    # Convert power
    #

    p=pow2dec[spot_tele_power]
    sum1=c1*26*26*26
    sum2=c2*26*26
    sum3=c3*26
    sum4=c4
    sum1_tot=sum1+sum2+sum3+sum4

    sum1=l1*18*10*10*19
    sum2=l2*10*10*19
    sum3=l3*10*19
    sum4=l4*19
    sum2_tot=sum1+sum2+sum3+sum4+p
    # print("sum_tot1/2:", sum1_tot,sum2_tot)

    # 24*1068
    lsub1=int(sum1_tot/25632)
    lsub2_tmp=sum1_tot-lsub1*25632
    lsub2=int(lsub2_tmp/1068)

    # print("lsub1/2",lsub1,lsub2)

    alt=(lsub2_tmp-lsub2*1068)*20

    # Handle bogus altitudes
    if  alt > 14000:
 #       print("Bogus packet. Too high altitude!! locking to 9999")
        alt=9999
        
    if alt == 2760:
 #       print("Bogus packet. 2760 m  locking to 9998")
        alt=9998

    if alt == 0:
#        print("Zero alt detected. Locking to 10000")
        alt=10000

    # Sublocator
    lsub1=lsub1+65
    lsub2=lsub2+65
    subloc=(chr(lsub1)+chr(lsub2)).lower()

    # Temperature
    # 40*42*2*2
    temp_1=int(sum2_tot/6720)
    temp_2=temp_1*2+457
    temp_3=temp_2*5/1024
    temp=(temp_2*500/1024)-273

    # print("Temp: %5.2f %5.2f %5.2f %5.2f" % (temp_1, temp_2, temp_3, temp))
    
    #
    # Battery
    #
    # =I7-J7*(40*42*2*2)
    batt_1=int(sum2_tot-temp_1*6720)
    batt_2=int(batt_1/168)
    batt_3=batt_2*10+614

    # 5*M8/1024
    batt=batt_3*5/1024

    #
    # Speed / GPS / Sats
    #
    # =I7-J7*(40*42*2*2)
    # =INT(L7/(42*2*2))
    t1=sum2_tot-temp_1*6720
    t2=int(t1/168)
    t3=t1-t2*168
    t4=int(t3/4)
    speed=t4*2
    r7=t3-t4*4
    gps=int(r7/2)
    sats=r7%2

    # print("T1-4,R7:",t1, t2, t3, t4, r7)

    #
    # Calc lat/lon from loc+subbloc
    #
    loc=spot_pos_loc+subloc
    lat,lon = (maidenhead.toLoc(loc))
    
    pstr =    ("Spot %s Call: %6s Latlon: %10.5f %10.5f Loc: %6s Alt: %5d Temp: %4.1f Batt: %5.2f Speed: %3d GPS: %1d Sats: %1d" %
          (  spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ))
    logging.info(pstr)

    telemetry = {'time':spot_pos_time, "call":spot_pos_call, "lat":lat, "lon":lon, "loc":loc, "alt": alt,
                 "temp":round(temp,1), "batt":round(batt,3), "speed":speed, "gps":gps, "sats":sats }

    return telemetry

# Check if sencence is in history of sent spots
def checkifsentdb(sentence):
    con = None
    try:
        con = sqlite3.connect('wsprdb.db')
        cur = con.cursor()
        cur.execute('select * from sentspots where sentstr=?', (sentence,))
        data = cur.fetchall()

#        for row in data:
#            print("found", row)
        if len(data) > 0:
            if con:
                con.close()
            return True
        if not data:
            con.commit()
    except sqlite3.Error as e:
        print("Database error: %s" % e)
    except Exception as e:
        print("Exception in _query: %s" % e)
    finally:
        if con:
            con.close()

    return False


def addsentdb(name, time_rec, sentence):
    con = None
    time_sent = datetime.datetime.now()
    
    try:
        con = sqlite3.connect('wsprdb.db')
        cur = con.cursor()
#         cur.execute('drop table if exists sentspots')
        cur.execute('create table if not exists sentspots(name varchar(15),time_sent varchar(20), time_received varchar(20), sentstr varchar(50))')
        cur.execute("INSERT INTO sentspots VALUES(?,?,?,?)", (name, time_sent, time_rec, sentence))
        data = cur.fetchall()
        if not data:
            con.commit()
    except sqlite3.Error as e:
        print("Database error: %s" % e)
    except Exception as e:
        print("Exception in _query: %s" % e)
    finally:
        if con:
            con.close()

    return




def send_tlm_to_habitat2(sentence, callsign):
    result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"])
    return

def send_tlm_to_habitat(sentence, callsign, spot_time):
    input=sentence

    logging.info("Pushing data to habhub")


    if not sentence.endswith('\n'):
        sentence += '\n'
    sentence = sentence.encode("utf-8")
    sentence2 = b64encode(sentence)
    sentence = str(sentence2,'utf-8')

#    callsign = sys.argv[2] if len(sys.argv) > 2 else "HABTOOLS"

    date_created = spot_time.isoformat("T") + "Z" 
    date = datetime.datetime.utcnow().isoformat("T") + "Z"

    data = {
        "type": "payload_telemetry",
        "data": {
            "_raw": sentence
            },
        "receivers": {
            callsign: {
                "time_created": date_created,
                "time_uploaded": date,
                },
            },
        }

    h = httplib2.Http("")

    resp, content = h.request(
        uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(),
        method='PUT',
        headers={'Content-Type': 'application/json; charset=UTF-8'},
        body=json.dumps(data),
    )

    # print(resp['status'])
    if resp['status'] == '201':
        logging.info("OK 201")
    elif resp['status'] == '403':
        logging.info("Error 403 - already uploaded")

    return


#
# Trim off older spots. Assuming oldest first!
#
def timetrim(spots, m):
    if len(spots) == 0:
        return spots

#    print("First:", spots[0][0], "Last: ", spots[-1:][0])
    time_last = datetime.datetime.utcnow() - timedelta(minutes=m)
    spotc = 0
    splitspotc = 0

    # find splitpoint in list
    for r in spots:
        spotc += 1
#        print(r[0], "vs ", time_last)
        if r[0] < time_last:
            splitspotc = spotc

    l = len(spots)
    if splitspotc > 0:
        spots = spots[splitspotc:]
        
    return spots


#
# Main function - filter, process and upload of telemetry
#
def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html):

    # Filter out telemetry-packets
    spots_tele = []
    for row in spots:
        # print(row)
        if re.match('(^0|^1|^Q).[0-9].*', row[1]):
            #         print(', '.join(row))
            #if re.match('10\..*', row[2]) or re.match('14\..*', row[2]):
            spots_tele.append(row)

    # 2018-05-03 13:06:00, QA5IQA, 7.040161, -8, JO53, 27, DH5RAE, JN68qv, 537
    # 0                    1       2         3   4     5   6       7       8 

    for b in balloons:
        if len(spots) == 0:
            logging.info("out of spots in balloonloop. returning")
            return spots
            
        balloon_name = b[0]
        balloon_call = b[1]
        balloon_mhz = b[2]
        balloon_channel = b[3]
        balloon_timeslot = b[4]
        balloon_html_push = b[6]
        balloon_ssid = b[7]
        balloon_aprs_comment = b[8]
        
        logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot))

        # Filter out telemetry for active channel
        if balloon_channel < 10:
            telem = [element for element in spots_tele if re.match('^0.'+str(balloon_channel), element[1])]
        else:
            telem = [element for element in spots_tele if re.match('^Q.'+str(balloon_channel-10), element[1])]

        # Filter out only selected band
        telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', element[2])]

        # If timeslot is used. filter out correct slot
        if balloon_timeslot > 0:
            telem = [element for element in telem if balloon_timeslot == int(element[0].minute % 10 / 2)]
                                
        #if len(telem) > 0:
            # logging.info("time at top-tele spot: %s", str(telem[0]))

        # Loop through all spots and try to find matching telemetrypacket
        spot_last = spots[0]
        spot_oldtime = datetime.datetime(1970, 1, 1, 1, 1)
        bspots = []
        for row in spots:
            if balloon_call == row[1]:
                bspots.append(row)
                
        logging.info("Spots: %d  Telemetry: %d", len(bspots), len(telem))
        for row in bspots:
            # logging.info("B: %s",row)
            spot_call = row[1]
            
            if balloon_call == spot_call:
                spot_time = row[0]

                # Only check new uniq times
                if spot_time != spot_oldtime:

                    # [datetime.datetime(2019, 8, 1, 0, 22), 'YO3ICT', '14.097148', -23, -1, 'BM73', 10, 'ND7M', 'DM16', 2564]
                    pstr = "Call: %s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" %  (row[1], row[0], row[2], row[5], row[6], row[7])
                    logging.info(pstr)
                    spot_oldtime = spot_time
                    spot_fq = row[2]
                    spot_power = row[4]
                    spot_loc = row[5]
                    spot_reporter = row[6]        
                    spot_reporter_loc = row[7]
                    
                # Match positioningpacket with telemetrypackets
                    b_telem = []
                    for trow in telem:
                        # print("tdiff:",trow, spot_time, type(spot_time))
                        tdiff = trow[0] - spot_time
                        #                    print(tdiff)

                        if tdiff > timedelta(minutes=8):
                            logging.info("Too long interval, breaking %s", str(tdiff))
                            break
                        if tdiff > timedelta(minutes=0):
                            b_telem.append(trow)

                    # If suitable telemetry found, enter decoding!
                    if len(b_telem) > 0:
                        logging.info("Found suitable telemetry rows: %d",len(b_telem))

                        # print("call", spot_call,"time",spot_time,"fq",spot_fq,"loc", spot_loc,"power",spot_power,"reporter",spot_reporter)
                        #logging.info(pstr)
                        #logging.info("T: %s", b_telem[0])
                        telemetry = decode_telemetry(row, b_telem[0])
                        if len(telemetry) > 0:
                            # Delete spot and telemetryspot
                            try:
                                spots.remove(row)
                            except ValueError:
                                pass

                            for rt in b_telem:
                                pstr = "%s Time: %s Fq: %s Loc: %s Power: %s Reporter: %s" %  (rt[1], rt[0], rt[2], rt[5], rt[6], rt[7]) 
                                logging.info("Removing: %s", pstr)
                                try:
                                    spots.remove(rt)
                                except ValueError:
                                    pass

                                try:
                                    spots_tele.remove(rt)
                                except ValueError:
                                    pass

                                try:
                                    telem.remove(rt)
                                except ValueError:
                                    pass

                            # logging.info(telemetry)

                            # seqnr = int(((int(telemetry['time'].strftime('%s'))) / 120) % 100000)
                            seqnr = int(telemetry['time'].strftime('%s'))

                            # telemetry = [ spot_pos_time, spot_pos_call, lat, lon, loc, alt, temp, batt, speed, gps, sats ]
                            telestr = "%s,%d,%s,%.5f,%.5f,%d,%d,%.2f,%.2f,%d,%d" % (  
                                balloon_name, seqnr, telemetry['time'].strftime('%H:%M'), telemetry['lat'], telemetry['lon'],
                                telemetry['alt'], telemetry['speed'], telemetry['temp'], telemetry['batt'], telemetry['gps'], telemetry['sats'])

                            # Calculate and add XOR-checksum
                            i=0
                            checksum = 0
                            while i < len(telestr):
                                checksum = checksum ^ ord(telestr[i])
                                i+=1
                            telestr = "$$" + telestr + "*" + '{:x}'.format(int(checksum))
                            logging.info("Telemetry: %s", telestr)

                            # Check if string has been uploaded before and if not then add and upload
                            if not checkifsentdb(telestr):
                                # print("Unsent spot", telestr)

                                print(push_habhub)
                                # push_habhub = True
                                if push_habhub:
                                    # Send telemetry to habhub
                                    send_tlm_to_habitat(telestr, habhub_callsign, spot_time)
                                else:
                                    logging.info("Not pushing to habhub")

                                if push_aprs:
                                    # Prep and push basic data to aprs.fi
                                    sonde_data = {}
                                    sonde_data["id"] = spot_call + "-" + balloon_ssid
                                    sonde_data["lat"] = telemetry['lat']
                                    sonde_data["lon"] = telemetry['lon']
                                    sonde_data["alt"] = telemetry['alt']
                                    sonde_data["speed"] = telemetry['speed']
                                    sonde_data["temp"] = telemetry['temp']
                                    sonde_data["batt"] = telemetry['batt']
                                    sonde_data["comment"] = balloon_aprs_comment
                                    logging.info("Pushing data to aprs.fi")
                                    push_balloon_to_aprs(sonde_data)
                                else:
                                    logging.info("Not pushing to aprs.fi")

                                if push_html and balloon_html_push:                                    
                                    # Push basic data to ftp html
                                    logging.info('\033[33m' + "Pushing data to html page" + '\033[0m')
                                    push_balloon_to_html(telemetry)			 

                                    
                                # Add sent string to history-db
                                addsentdb(balloon_name, row[0], telestr)

                            else:
                                logging.info("Already sent spot. Doing nothing")

    return spots