Add stress tester that simulates super high chance of BLE failures

and fix bug that it revealed (missing handling of error during discovery)

TODO.md

@@ -1,5 +1,6 @@
 # Remaining tasks before declaring 1.0
 
+- add faq entry about range and antennas and rain
 - first message sent is still doubled for some people
 - disable software update button after update finishes
 - let users set arbitrary params in android

app/src/main/java/com/geeksville/mesh/service/BluetoothInterface.kt

@@ -354,9 +354,9 @@ class BluetoothInterface(val service: RadioInterfaceService, val address: String
     private var isFirstTime = true
 
     private fun doDiscoverServicesAndInit() {
-        // FIXME - no need to discover services more than once - instead use lazy() to use them in future attempts
         safe!!.asyncDiscoverServices { discRes ->
-            discRes.getOrThrow() // FIXME, instead just try to reconnect?
+            try {
+                discRes.getOrThrow()
 
                 service.serviceScope.handledLaunch {
                     try {
@@ -384,6 +384,11 @@ class BluetoothInterface(val service: RadioInterfaceService, val address: String
                         )
                     }
                 }
+            } catch (ex: BLEException) {
+                scheduleReconnect(
+                    "Unexpected error discovering services, forcing disconnect $ex"
+                )
+            }
         }
     }
 

app/src/main/java/com/geeksville/mesh/service/SafeBluetooth.kt

@@ -85,8 +85,11 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
         }
 
         override fun toString(): String {
-            return super.toString()
+            return "Work:$tag"
         }
+
+        /// Connection work items are treated specially
+        fun isConnect() = tag == "connect" || tag == "reconnect"
     }
 
     /**
@@ -122,6 +125,7 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
     private val STATUS_RELIABLE_WRITE_FAILED = 4403
     private val STATUS_TIMEOUT = 4404
     private val STATUS_NOSTART = 4405
+    private val STATUS_SIMFAILURE = 4406
 
     private val gattCallback = object : BluetoothGattCallback() {
 
@@ -157,6 +161,10 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
                         if (oldstate == BluetoothProfile.STATE_CONNECTED) {
                             info("Lost connection - aborting current work: $currentWork")
 
+                            // If we get a disconnect, just try again otherwise fail all current operations
+                            if (currentWork?.isConnect() == true)
+                                dropAndReconnect()
+                            else
                                 lostConnection("lost connection")
                         } else if (status == 133) {
                             // We were not previously connected and we just failed with our non-auto connection attempt.  Therefore we now need
@@ -285,6 +293,12 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
         }
     }
 
+    // To test loss of BLE faults we can randomly fail a certain % of all work items.  We
+    // skip this for "connect" items because the handling for connection failure is special
+    var simFailures = false
+    var failPercent =
+        10 // 15% failure is unusably high because of constant reconnects, 7% somewhat usable, 10% pretty bad
+    private val failRandom = Random()
 
     private var activeTimeout: Job? = null
 
@@ -311,6 +325,14 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
 
             isSettingMtu =
                 false // Most work is not doing MTU stuff, the work that is will re set this flag
+
+            val failThis =
+                simFailures && !newWork.isConnect() && failRandom.nextInt(100) < failPercent
+
+            if (failThis) {
+                errormsg("Simulating random work failure!")
+                completeWork(STATUS_SIMFAILURE, Unit)
+            } else {
                 val started = newWork.startWork()
                 if (!started) {
                     errormsg("Failed to start work, returned error status")
@@ -321,6 +343,7 @@ class SafeBluetooth(private val context: Context, private val device: BluetoothD
                 }
             }
         }
+    }
 
     private fun <T> queueWork(
         tag: String,

geeksville-androidlib

@@ -1 +1 @@
-Subproject commit 2f9243202d9db33b0aa0c7656bc8916ad3712914
+Subproject commit 95c5a9aa950f917857a3cc0c7cd84a4a56993032