diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d5123..5afe70e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,17 @@ The following is a history of the changes made to this project. -## v2.7.1 *(pre-release)* +## v2.7.2 *(prerelease)* + +* Updated current PiAware and dump1090-fa versions to 3.9.3. +* When installing dump1090-fa the maintainer packages for libbladerf are used. +* Removed installation restriction for dump1090-fa on Ubuntu. +* Fixed reported JSON incompatabilities with dump1090-fa. *(advanced)* +* Removed Flightradar24 ARM client version from variables file. +* Fixed issue pertaining to theremoval of old style ADS-B Exchange rc.local enties. +* Added indexes to tables to improve performance. (thanks to @target-drone) *(advanced)* + +## v2.7.1 *(not released)* * Added missing purgeAircraft setting to the database. *(advanced)* diff --git a/bash/decoders/dump1090-fa.sh b/bash/decoders/dump1090-fa.sh index 49638c0..e60f42d 100755 --- a/bash/decoders/dump1090-fa.sh +++ b/bash/decoders/dump1090-fa.sh @@ -81,90 +81,16 @@ fi echo -e "\e[95m Installing packages needed to build and fulfill dependencies...\e[97m" echo "" -CheckPackage git -CheckPackage curl -CheckPackage build-essential CheckPackage debhelper -CheckPackage cron -CheckPackage rtl-sdr CheckPackage librtlsdr-dev -CheckPackage libusb-1.0-0 CheckPackage libusb-1.0-0-dev -CheckPackage libtecla1 CheckPackage pkg-config -CheckPackage lighttpd -CheckPackage fakeroot CheckPackage dh-systemd CheckPackage libncurses5-dev -CheckPackage cmake -CheckPackage doxygen -CheckPackage libtecla-dev -CheckPackage help2man -CheckPackage pandoc - -## BUILD AND INSTALL THE BLADERF PACKAGE FROM SOURCE IF NOT INSTALLED - -# Check if the needed bladeRF packages are installed. -if [[ $(dpkg-query -W -f='${STATUS}' libbladerf2 2>/dev/null | grep -c "ok installed") -eq 0 ]] || [[ $(dpkg-query -W -f='${STATUS}' libbladerf-dev 2>/dev/null | grep -c "ok installed") -eq 0 ]] || [[ $(dpkg-query -W -f='${STATUS}' libbladerf-udev 2>/dev/null | grep -c "ok installed") -eq 0 ]]; then - echo "" - echo -e "\e[95m Preparing the bladeRF Git repository...\e[97m" - echo "" - if [[ -d "${RECEIVER_BUILD_DIRECTORY}/bladeRF/bladeRF" ]] && [[ -d "${RECEIVER_BUILD_DIRECTORY}/bladeRF/bladeRF/.git" ]] ; then - # A directory with a git repository containing the source code already exists. - echo -e "\e[94m Entering the bladeRF git repository directory...\e[97m" - cd ${RECEIVER_BUILD_DIRECTORY}/bladeRF/bladeRF 2>&1 - echo -e "\e[94m Updating the local bladeRF git repository...\e[97m" - echo "" - git pull - else - # A directory containing the source code does not exist in the build directory. - echo -e "\e[94m Creating the bladeRF build directory...\e[97m" - echo "" - mkdir -vp ${RECEIVER_BUILD_DIRECTORY}/bladeRF - echo "" - cd ${RECEIVER_BUILD_DIRECTORY}/bladeRF 2>&1 - echo -e "\e[94m Cloning the bladeRF git repository locally...\e[97m" - echo "" - git clone https://github.com/Nuand/bladeRF.git - fi - - echo "" - echo -e "\e[95m Building and installing the bladeRF package...\e[97m" - echo "" - if [[ ! "${PWD}" = "${RECEIVER_BUILD_DIRECTORY}/bladeRF/bladeRF" ]] ; then - echo -e "\e[94m Entering the bladeRF git repository directory...\e[97m" - cd ${RECEIVER_BUILD_DIRECTORY}/bladeRF/bladeRF 2>&1 - fi - - echo -e "\e[94m Building the bladeRF package...\e[97m" - echo "" - dpkg-buildpackage -b - echo "" - - echo -e "\e[94m Entering the bladeRF build directory...\e[97m" - cd ${RECEIVER_BUILD_DIRECTORY}/bladeRF 2>&1 - - if [[ $(dpkg-query -W -f='${STATUS}' libbladerf2 2>/dev/null | grep -c "ok installed") -eq 0 ]]; then - echo -e "\e[94m Installing the libbladerf2 package...\e[97m" - echo "" - sudo dpkg -i libbladerf2_*.deb - echo "" - fi - - if [[ $(dpkg-query -W -f='${STATUS}' libbladerf-dev 2>/dev/null | grep -c "ok installed") -eq 0 ]]; then - echo -e "\e[94m Installing the libbladerf-dev package...\e[97m" - echo "" - sudo dpkg -i libbladerf-dev_*.deb - echo "" - fi - - if [[ $(dpkg-query -W -f='${STATUS}' libbladerf-udev 2>/dev/null | grep -c "ok installed") -eq 0 ]]; then - echo -e "\e[94m Installing the libbladerf-udev package...\e[97m" - echo "" - sudo dpkg -i libbladerf-udev_*.deb - fi -fi -echo "" +CheckPackage libbladerf1 +CheckPackage libbladerf-dev +CheckPackage adduser +CheckPackage lighttpd ## DOWNLOAD OR UPDATE THE DUMP1090-FA SOURCE diff --git a/bash/main.sh b/bash/main.sh index 00ddb61..dbfb898 100755 --- a/bash/main.sh +++ b/bash/main.sh @@ -286,14 +286,7 @@ fi if [[ ! "${DUMP1090_IS_INSTALLED}" = "true" ]] ; then # If this is not an automated installation ask the user which one to install. if [[ ! "${RECEIVER_AUTOMATED_INSTALL}" = "true" ]] ; then - - # Do not show dump 1090-fa option for Ubuntu 17.10 or higher until it is updated to support it. - if [ ! "$RECEIVER_OS_DISTRIBUTION" == "ubuntu" ] && (( $(bc -l <<<"$RECEIVER_OS_RELEASE < 17.10") )); then - DUMP1090_OPTION=$(whiptail --nocancel --backtitle "${RECEIVER_PROJECT_TITLE}" --title "Choose Dump1090 Version To Install" --radiolist "Dump1090 does not appear to be present on this device. In order to continue setup dump1090 will need to exist on this device. Please select your prefered dump1090 version from the list below.\n\nPlease note that in order to run dump1090-fa PiAware will need to be installed as well." 16 65 3 "dump1090-mutability" "(Mutability)" ON "dump1090-fa" "(FlightAware)" OFF "dump1090-hptoa" "(OpenSky Network)" OFF 3>&1 1>&2 2>&3) - else - DUMP1090_OPTION=$(whiptail --nocancel --backtitle "${RECEIVER_PROJECT_TITLE}" --title "Choose Dump1090 Version To Install" --radiolist "Dump1090 does not appear to be present on this device. In order to continue setup dump1090 will need to exist on this device. Please select your prefered dump1090 version from the list below.\n\nPlease note that in order to run dump1090-fa PiAware will need to be installed as well." 16 65 3 "dump1090-mutability" "(Mutability)" ON "dump1090-hptoa" "(OpenSky Network)" OFF 3>&1 1>&2 2>&3) - fi - + DUMP1090_OPTION=$(whiptail --nocancel --backtitle "${RECEIVER_PROJECT_TITLE}" --title "Choose Dump1090 Version To Install" --radiolist "Dump1090 does not appear to be present on this device. In order to continue setup dump1090 will need to exist on this device. Please select your prefered dump1090 version from the list below.\n\nPlease note that in order to run dump1090-fa PiAware will need to be installed as well." 16 65 3 "dump1090-mutability" "(Mutability)" ON "dump1090-fa" "(FlightAware)" OFF "dump1090-hptoa" "(OpenSky Network)" OFF 3>&1 1>&2 2>&3) case ${DUMP1090_OPTION} in "dump1090-mutability") DUMP1090_FORK="mutability" diff --git a/bash/portal/install.sh b/bash/portal/install.sh index 295a4ee..2285ea1 100755 --- a/bash/portal/install.sh +++ b/bash/portal/install.sh @@ -271,6 +271,7 @@ CheckPackage libpython2.7 # Install packages needed for advanced portal setups. if [[ "${ADVANCED}" = "true" ]] ; then CheckPackage python-pyinotify + CheckPackage python-apt case "${DATABASEENGINE}" in "MySQL") CheckPackage mysql-client diff --git a/bash/portal/logging.sh b/bash/portal/logging.sh index 88eebdf..55fd651 100755 --- a/bash/portal/logging.sh +++ b/bash/portal/logging.sh @@ -79,6 +79,13 @@ echo -e "" echo -e "\e[95m Setting up flight logging...\e[97m" echo -e "" +# Dump1090-fa has changed the structure of their JSON and needed a new version of flights.py. +if [ ] ; then + FLIGHTS_FILE='flights.fa.py' +else + FLIGHTS_FILE='flights.mutability.py' +fi + # Create and set permissions on the flight logging and maintenance maintenance scripts. echo -e "\e[94m Creating the flight logging maintenance script...\e[97m" tee ${PORTAL_PYTHON_DIRECTORY}/flights-maint.sh > /dev/null < /dev/null <pdoOpen(); + $sql = "CREATE INDEX IF NOT EXISTS idxIcao ON ".$settings::db_prefix."aircraft(icao)"; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth = NULL; + $dbh = NULL; + + // Add an index to the positions table. + $dbh = $common->pdoOpen(); + $sql = "CREATE INDEX IF NOT EXISTS idxFlight ON ".$settings::db_prefix."positions(flight)"; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth = NULL; + $dbh = NULL; + } + + // Update the version and patch settings.. + $common->updateSetting("version", "2.7.2"); + $common->updateSetting("patch", ""); + + // The upgrade process completed successfully. + $results['success'] = TRUE; + $results['message'] = "Upgrade to v2.7.2 successful."; + return $results; + + } catch(Exception $e) { + // Something went wrong during this upgrade process. + $results['success'] = FALSE; + $results['message'] = $e->getMessage(); + return $results; + } + } +?> + diff --git a/build/portal/html/install/upgrade.php b/build/portal/html/install/upgrade.php index 7e2f713..7aaf029 100644 --- a/build/portal/html/install/upgrade.php +++ b/build/portal/html/install/upgrade.php @@ -33,7 +33,7 @@ $common = new common(); // The most current stable release. - $thisVersion = "2.7.1"; + $thisVersion = "2.7.2"; // Begin the upgrade process if this release is newer than what is installed. if ($common->getSetting("version") == $thisVersion) { @@ -168,6 +168,15 @@ $version = "2.7.1"; } + // UPGRADE TO V2.7.2 + if ($common->getSetting("version") == "2.7.1" && $success) { + $json = file_get_contents("http://localhost/install/upgrade-v2.7.2.php"); + $results = json_decode($json, TRUE); + $success = $results['success']; + $message = $results['message']; + $version = "2.7.2"; + } + require_once($_SERVER['DOCUMENT_ROOT'].DIRECTORY_SEPARATOR."admin".DIRECTORY_SEPARATOR."includes".DIRECTORY_SEPARATOR."header.inc.php"); // Display the instalation wizard. diff --git a/build/portal/python/flights.fa.py b/build/portal/python/flights.fa.py new file mode 100644 index 0000000..cf98511 --- /dev/null +++ b/build/portal/python/flights.fa.py @@ -0,0 +1,209 @@ +#!/usr/bin/python + +#================================================================================# +# ADS-B FEEDER PORTAL # +# ------------------------------------------------------------------------------ # +# Copyright and Licensing Information: # +# # +# The MIT License (MIT) # +# # +# Copyright (c) 2015-2016 Joseph A. Prochazka # +# # +# Permission is hereby granted, free of charge, to any person obtaining a copy # +# of this software and associated documentation files (the "Software"), to deal # +# in the Software without restriction, including without limitation the rights # +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # +# copies of the Software, and to permit persons to whom the Software is # +# furnished to do so, subject to the following conditions: # +# # +# The above copyright notice and this permission notice shall be included in all # +# copies or substantial portions of the Software. # +# # +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # +# SOFTWARE. # +#================================================================================# + +# WHAT THIS DOES: +# --------------------------------------------------------------- +# +# 1) Read aircraft.json generated by dump1090-mutability. +# 2) Add the flight to the database if it does not already exist. +# 3) Update the last time the flight was seen. + +import datetime +import json +import time +import os +#urllib2 is deprecated in python3 +try: + # For Python 3.0 and later + from urllib.request import urlopen +except ImportError: + # Fall back to Python 2's urllib2 + from urllib2 import urlopen + +def log(string): + #print(string) # uncomment to enable debug logging + return + +# Read the configuration file. +with open(os.path.dirname(os.path.realpath(__file__)) + '/config.json') as config_file: + config = json.load(config_file) + +# Import the needed database library. +if config["database"]["type"] == "mysql": + import MySQLdb +else: + import sqlite3 + +class FlightsProcessor(object): + def __init__(self, config): + self.config = config + self.dbType = config["database"]["type"] + # List of required keys for position data entries + self.position_keys = ('lat', 'lon', 'nav_altitude', 'gs', 'track', 'geom_rate', 'hex') + + def setupDBStatements(self, formatSymbol): + if hasattr(self, 'STMTS'): + return + mapping = { "s": formatSymbol } + self.STMTS = { + 'select_aircraft_count':"SELECT COUNT(*) FROM adsb_aircraft WHERE icao = %(s)s" % mapping, + 'select_aircraft_id': "SELECT id FROM adsb_aircraft WHERE icao = %(s)s" % mapping, + 'select_flight_count': "SELECT COUNT(*) FROM adsb_flights WHERE flight = %(s)s" % mapping, + 'select_flight_id': "SELECT id FROM adsb_flights WHERE flight = %(s)s" % mapping, + 'select_position': "SELECT message FROM adsb_positions WHERE flight = %(s)s AND message = %(s)s ORDER BY time DESC LIMIT 1" % mapping, + 'insert_aircraft': "INSERT INTO adsb_aircraft (icao, firstSeen, lastSeen) VALUES (%(s)s, %(s)s, %(s)s)" % mapping, + 'insert_flight': "INSERT INTO adsb_flights (aircraft, flight, firstSeen, lastSeen) VALUES (%(s)s, %(s)s, %(s)s, %(s)s)" % mapping, + 'insert_position_sqwk': "INSERT INTO adsb_positions (flight, time, message, squawk, latitude, longitude, track, altitude, verticleRate, speed, aircraft) VALUES (%(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s)" % mapping, + 'insert_position': "INSERT INTO adsb_positions (flight, time, message, latitude, longitude, track, altitude, verticleRate, speed, aircraft) VALUES (%(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s, %(s)s)" % mapping, + 'update_aircraft_seen': "UPDATE adsb_aircraft SET lastSeen = %(s)s WHERE icao = %(s)s" % mapping, + 'update_flight_seen': "UPDATE adsb_flights SET aircraft = %(s)s, lastSeen = %(s)s WHERE flight = %(s)s" % mapping + } + + def connectDB(self): + if self.dbType == "sqlite": ## Connect to a SQLite database. + self.setupDBStatements("?") + return sqlite3.connect(self.config["database"]["db"]) + else: ## Connect to a MySQL database. + self.setupDBStatements("%s") + return MySQLdb.connect(host=self.config["database"]["host"], + user=self.config["database"]["user"], + passwd=self.config["database"]["passwd"], + db=self.config["database"]["db"]) + + def processAircraftList(self, aircraftList): + db = self.connectDB() + # Get Database cursor handle + self.cursor = db.cursor() + # Assign the time to a variable. + self.time_now = datetime.datetime.utcnow().strftime("%Y/%m/%d %H:%M:%S") + + for aircraft in aircraftList: + self.processAircraft(aircraft) + + # Close the database connection. + db.commit() + db.close() + + def processAircraft(self, aircraft): + hexcode = aircraft["hex"] + # Check if this aircraft was already seen. + self.cursor.execute(self.STMTS['select_aircraft_count'], (hexcode,)) + row_count = self.cursor.fetchone() + if row_count[0] == 0: + # Insert the new aircraft. + log("Added Aircraft: " + hexcode) + self.cursor.execute(self.STMTS['insert_aircraft'], (hexcode, self.time_now, self.time_now,)) + else: + # Update the existing aircraft. + self.cursor.execute(self.STMTS['update_aircraft_seen'], (self.time_now, hexcode,)) + log("Updating Aircraft: " + hexcode) + # Get the ID of this aircraft. + self.cursor.execute(self.STMTS['select_aircraft_id'], (hexcode,)) + row = self.cursor.fetchone() + aircraft_id = row[0] + log("\tFound Aircraft ID: " + str(aircraft_id)) + + # Check that a flight is tied to this track. + if aircraft.has_key('flight'): + self.processFlight(aircraft_id, aircraft) + + def processFlight(self, aircraft_id, aircraft): + flight = aircraft["flight"].strip() + # Check to see if the flight already exists in the database. + self.cursor.execute(self.STMTS['select_flight_count'], (flight,)) + row_count = self.cursor.fetchone() + if row_count[0] == 0: + # If the flight does not exist in the database add it. + params = (aircraft_id, flight, self.time_now, self.time_now,) + self.cursor.execute(self.STMTS['insert_flight'], params) + log("\t\tAdded Flight: " + flight) + else: + # If it already exists pdate the time it was last seen. + params = (aircraft_id, self.time_now, flight,) + self.cursor.execute(self.STMTS['update_flight_seen'], params) + log("\t\tUpdated Flight: " + flight) + # Get the ID of this flight. + self.cursor.execute(self.STMTS['select_flight_id'], (flight,)) + row = self.cursor.fetchone() + flight_id = row[0] + + # Check if position data is available. + if (all (k in aircraft for k in self.position_keys) and aircraft["altitude"] != "ground"): + self.processPositions(flight_id, aircraft) + + def processPositions(self, flight_id, aircraft): + # Get the ID of this aircraft. + hexcode = aircraft["hex"] + self.cursor.execute(self.STMTS['select_aircraft_id'], (hexcode,)) + row = self.cursor.fetchone() + aircraft_id = row[0] + + # Check that this message has not already been added to the database. + params = (flight_id, aircraft["messages"],) + self.cursor.execute(self.STMTS['select_position'], params) + row = self.cursor.fetchone() + + if row == None or row[0] != aircraft["messages"]: + # Add this position to the database. + if aircraft.has_key('squawk'): + params = (flight_id, self.time_now, aircraft["messages"], aircraft["squawk"], + aircraft["lat"], aircraft["lon"], aircraft["track"], + aircraft["nav_altitude"], aircraft["geom_rate"], aircraft["gs"], aircraft_id,) + self.cursor.execute(self.STMTS['insert_position_sqwk'], params) + log("\t\t\tInserted position w/ Squawk " + repr(params)) + else: + params = (flight_id, self.time_now, aircraft["messages"], aircraft["lat"], aircraft["lon"], + aircraft["track"], aircraft["nav_altitude"], aircraft["geom_rate"], aircraft["gs"], aircraft_id,) + self.cursor.execute(self.STMTS['insert_position'], params) + log("\t\t\tInserted position w/o Squawk " + repr(params)) + else: + log("\t\t\tMessage is the same") + + +if __name__ == "__main__": + processor = FlightsProcessor(config) + + # Main run loop + while True: + # Read dump1090-mutability's aircraft.json. + #with open('/run/dump1090-mutability/aircraft.json') as data_file: + # data = json.load(data_file) + + # Switch from physical file location to using urlopen after the addition of the dump1090-fa option. + # dump1090-fa and dump1090-mutability store aircraft.json in difrent locations. + # However Lighttpd is set up to serve this file using the same URL no matter which version is installed. + response = urlopen('http://localhost/dump1090/data/aircraft.json') + data = json.load(response) + + processor.processAircraftList(data["aircraft"]) + + log("Last Run: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")) + time.sleep(15) + diff --git a/build/portal/python/flights.py b/build/portal/python/flights.mutability.py similarity index 100% rename from build/portal/python/flights.py rename to build/portal/python/flights.mutability.py