2021-10-08 19:18:52 +00:00
package org.signal.core.util.concurrent
import android.os.Handler
2021-11-03 18:22:53 +00:00
import org.signal.core.util.ExceptionUtil
2021-10-08 19:18:52 +00:00
import org.signal.core.util.logging.Log
2021-10-21 20:32:25 +00:00
import java.util.concurrent.ExecutorService
2021-10-08 19:18:52 +00:00
/ * *
* A class that polls active threads at a set interval and logs when multiple threads are BLOCKED .
* /
class DeadlockDetector ( private val handler : Handler , private val pollingInterval : Long ) {
private var running = false
2021-10-13 15:00:48 +00:00
private val previouslyBlocked : MutableSet < Long > = mutableSetOf ( )
2022-04-27 14:30:30 +00:00
private val waitingStates : Set < Thread . State > = setOf ( Thread . State . WAITING , Thread . State . TIMED _WAITING )
@Volatile
var lastThreadDump : Map < Thread , Array < StackTraceElement > > ? = null
@Volatile
var lastThreadDumpTime : Long = - 1
2021-10-08 19:18:52 +00:00
fun start ( ) {
2023-02-13 22:03:08 +00:00
Log . d ( TAG , " Beginning deadlock monitoring. " )
2021-10-08 19:18:52 +00:00
running = true
handler . postDelayed ( this :: poll , pollingInterval )
}
fun stop ( ) {
2023-02-13 22:03:08 +00:00
Log . d ( TAG , " Ending deadlock monitoring. " )
2021-10-08 19:18:52 +00:00
running = false
handler . removeCallbacksAndMessages ( null )
}
private fun poll ( ) {
2022-04-27 14:30:30 +00:00
val time : Long = System . currentTimeMillis ( )
2021-10-13 15:00:48 +00:00
val threads : Map < Thread , Array < StackTraceElement > > = Thread . getAllStackTraces ( )
2021-10-26 13:20:41 +00:00
val blocked : Map < Thread , Array < StackTraceElement > > = threads
. filter { entry ->
val thread : Thread = entry . key
val stack : Array < StackTraceElement > = entry . value
2022-07-05 14:38:21 +00:00
thread . state == Thread . State . BLOCKED || ( thread . state . isWaiting ( ) && stack . hasPotentialLock ( ) )
2021-10-26 13:20:41 +00:00
}
2022-04-27 14:30:30 +00:00
. filter { entry -> ! BLOCK_BLOCKLIST . contains ( entry . key . name ) }
2021-10-13 15:00:48 +00:00
val blockedIds : Set < Long > = blocked . keys . map ( Thread :: getId ) . toSet ( )
2021-10-21 20:32:25 +00:00
val stillBlocked : Set < Long > = blockedIds . intersect ( previouslyBlocked )
2021-10-08 19:18:52 +00:00
if ( blocked . size > 1 ) {
2021-10-13 15:00:48 +00:00
Log . w ( TAG , buildLogString ( " Found multiple blocked threads! Possible deadlock. " , blocked ) )
2022-04-27 14:30:30 +00:00
lastThreadDump = threads
lastThreadDumpTime = time
2021-10-21 20:32:25 +00:00
} else if ( stillBlocked . isNotEmpty ( ) ) {
val stillBlockedMap : Map < Thread , Array < StackTraceElement > > = stillBlocked
. map { blockedId ->
val key : Thread = blocked . keys . first { it . id == blockedId }
val value : Array < StackTraceElement > = blocked [ key ] !!
Pair ( key , value )
}
. toMap ( )
Log . w ( TAG , buildLogString ( " Found a long block! Blocked for at least $pollingInterval ms. " , stillBlockedMap ) )
2022-04-27 14:30:30 +00:00
lastThreadDump = threads
lastThreadDumpTime = time
2021-10-21 20:32:25 +00:00
}
2021-11-03 18:22:53 +00:00
val fullExecutors : List < ExecutorInfo > = CHECK_FULLNESS_EXECUTORS . filter { isExecutorFull ( it . executor ) }
2021-10-21 20:32:25 +00:00
if ( fullExecutors . isNotEmpty ( ) ) {
fullExecutors . forEach { executorInfo ->
val fullMap : Map < Thread , Array < StackTraceElement > > = threads
. filter { it . key . name . startsWith ( executorInfo . namePrefix ) }
2021-10-13 15:00:48 +00:00
. toMap ( )
2021-11-03 18:22:53 +00:00
val executor : TracingExecutorService = executorInfo . executor as TracingExecutorService
2021-10-23 02:33:41 +00:00
Log . w ( TAG , buildLogString ( " Found a full executor! ${executor.activeCount} / ${executor.maximumPoolSize} threads active with ${executor.queue.size} tasks queued. " , fullMap ) )
2021-10-13 15:00:48 +00:00
}
2022-04-27 14:30:30 +00:00
lastThreadDump = threads
lastThreadDumpTime = time
2021-10-08 19:18:52 +00:00
}
2021-10-13 15:00:48 +00:00
previouslyBlocked . clear ( )
previouslyBlocked . addAll ( blockedIds )
2021-10-08 19:18:52 +00:00
if ( running ) {
handler . postDelayed ( this :: poll , pollingInterval )
}
}
2021-10-21 20:32:25 +00:00
private data class ExecutorInfo (
val executor : ExecutorService ,
val namePrefix : String
)
2022-07-05 14:38:21 +00:00
private fun Thread . State . isWaiting ( ) : Boolean {
return waitingStates . contains ( this )
}
private fun Array < StackTraceElement > . hasPotentialLock ( ) : Boolean {
return any {
2022-08-19 18:17:45 +00:00
it . methodName . startsWith ( " lock " ) || ( it . methodName . startsWith ( " waitForConnection " ) && ! it . className . contains ( " IncomingMessageObserver " ) )
2022-07-05 14:38:21 +00:00
}
}
2021-10-08 19:18:52 +00:00
companion object {
private val TAG = Log . tag ( DeadlockDetector :: class . java )
2021-11-03 18:22:53 +00:00
private val CHECK _FULLNESS _EXECUTORS : Set < ExecutorInfo > = setOf (
2021-10-21 20:32:25 +00:00
ExecutorInfo ( SignalExecutors . BOUNDED , " signal-bounded- " ) ,
2021-10-22 15:50:21 +00:00
ExecutorInfo ( SignalExecutors . BOUNDED _IO , " signal-io-bounded " )
2021-10-21 20:32:25 +00:00
)
private const val CONCERNING _QUEUE _THRESHOLD = 4
2022-04-27 14:30:30 +00:00
private val BLOCK _BLOCKLIST = setOf ( " HeapTaskDaemon " )
2021-10-26 13:20:41 +00:00
2021-10-13 15:00:48 +00:00
private fun buildLogString ( description : String , blocked : Map < Thread , Array < StackTraceElement > > ) : String {
2021-10-08 19:18:52 +00:00
val stringBuilder = StringBuilder ( )
2021-10-13 15:00:48 +00:00
stringBuilder . append ( description ) . append ( " \n " )
2021-10-08 19:18:52 +00:00
for ( entry in blocked ) {
2021-10-14 14:57:35 +00:00
stringBuilder . append ( " -- [ ${entry.key.id} ] ${entry.key.name} | ${entry.key.state} \n " )
2021-10-08 19:18:52 +00:00
2021-11-03 18:22:53 +00:00
val callerThrowable : Throwable ? = TracedThreads . callerStackTraces [ entry . key . id ]
val stackTrace : Array < StackTraceElement > = if ( callerThrowable != null ) {
ExceptionUtil . joinStackTrace ( entry . value , callerThrowable . stackTrace )
} else {
entry . value
}
for ( element in stackTrace ) {
2021-10-08 19:18:52 +00:00
stringBuilder . append ( " $element \n " )
}
stringBuilder . append ( " \n " )
}
return stringBuilder . toString ( )
}
2021-10-21 20:32:25 +00:00
private fun isExecutorFull ( executor : ExecutorService ) : Boolean {
2021-11-03 18:22:53 +00:00
return if ( executor is TracingExecutorService ) {
2021-10-21 20:32:25 +00:00
executor . queue . size > CONCERNING _QUEUE _THRESHOLD
} else {
false
}
}
2021-10-08 19:18:52 +00:00
}
2021-10-21 20:32:25 +00:00
}