Merge pull request #365 from mcumings/fix-340

Fixes #340 Improve CSV file export

app/src/main/java/com/geeksville/mesh/MainActivity.kt

@@ -43,7 +43,6 @@ import com.geeksville.android.Logging
 import com.geeksville.android.ServiceClient
 import com.geeksville.concurrent.handledLaunch
 import com.geeksville.mesh.android.*
-import com.geeksville.mesh.database.entity.Packet
 import com.geeksville.mesh.databinding.ActivityMainBinding
 import com.geeksville.mesh.model.ChannelSet
 import com.geeksville.mesh.model.DeviceVersion
@@ -62,13 +61,14 @@ import com.google.android.material.snackbar.Snackbar
 import com.google.android.material.tabs.TabLayoutMediator
 import com.vorlonsoft.android.rate.AppRate
 import com.vorlonsoft.android.rate.StoreType
-import kotlinx.coroutines.*
-import java.io.FileOutputStream
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.cancel
 import java.nio.charset.Charset
 import java.text.DateFormat
 import java.util.*
 import java.util.regex.Pattern
-import kotlin.math.roundToInt
 
 
 /*
@@ -655,20 +655,7 @@ class MainActivity : AppCompatActivity(), Logging,
             }
             CREATE_CSV_FILE -> {
                 if (resultCode == Activity.RESULT_OK) {
-                    data?.data?.let { file_uri ->
-                        // model.allPackets is a result of a query, so we need to use observer for
-                        // the query to materialize
-                        model.allPackets.observe(this, { packets ->
-                            if (packets != null) {
-                                // no need for observer once got non-null list
-                                model.allPackets.removeObservers(this)
-                                // execute on the default thread pool to not block the main thread
-                                CoroutineScope(Dispatchers.Default + Job()).handledLaunch {
-                                    saveMessagesCSV(file_uri, packets)
-                                }
-                            }
-                        })
-                    }
+                    data?.data?.let { file_uri -> model.saveMessagesCSV(file_uri) }
                 }
             }
         }
@@ -1190,40 +1177,6 @@ class MainActivity : AppCompatActivity(), Logging,
         }
     }
 
-    private fun saveMessagesCSV(file_uri: Uri, packets: List<Packet>) {
-        // Extract distances to this device from position messages and put (node,SNR,distance) in
-        // the file_uri
-        val myNodeNum = model.myNodeInfo.value?.myNodeNum ?: return
-
-        applicationContext.contentResolver.openFileDescriptor(file_uri, "w")?.use {
-            FileOutputStream(it.fileDescriptor).use { fs ->
-                // Write header
-                fs.write(("from,rssi,snr,time,dist\n").toByteArray())
-                // Packets are ordered by time, we keep most recent position of
-                // our device in my_position.
-                var my_position: MeshProtos.Position? = null
-                packets.forEach {
-                    it.proto?.let { packet_proto ->
-                        it.position?.let { position ->
-                            if (packet_proto.from == myNodeNum) {
-                                my_position = position
-                            } else if (my_position != null) {
-                                val dist = positionToMeter(my_position!!, position).roundToInt()
-                                fs.write(
-                                    "%x,%d,%f,%d,%d\n".format(
-                                        packet_proto.from, packet_proto.rxRssi,
-                                        packet_proto.rxSnr, packet_proto.rxTime, dist
-                                    ).toByteArray()
-                                )
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
     /// Theme functions
 
     private fun chooseThemeDialog() {

app/src/main/java/com/geeksville/mesh/NodeInfo.kt

@@ -71,6 +71,13 @@ data class Position(
     /// @return bearing to the other position in degrees
     fun bearing(o: Position) = bearing(latitude, longitude, o.latitude, o.longitude)
 
+    // If GPS gives a crap position don't crash our app
+    fun isValid(): Boolean {
+        return (latitude <= 90.0 && latitude >= -90) &&
+                latitude != 0.0 &&
+                longitude != 0.0
+    }
+
     override fun toString(): String {
         return "Position(lat=${latitude.anonymize}, lon=${longitude.anonymize}, alt=${altitude.anonymize}, time=${time}, batteryPctLevel=${batteryPctLevel})"
     }
@@ -112,11 +119,7 @@ data class NodeInfo(
     /// return the position if it is valid, else null
     val validPosition: Position?
         get() {
-            return position?.takeIf {
-                (it.latitude <= 90.0 && it.latitude >= -90) && // If GPS gives a crap position don't crash our app
-                    it.latitude != 0.0 &&
-                    it.longitude != 0.0
-            }
+            return position?.takeIf { it.isValid() }
         }
 
     /// @return distance in meters to some other node (or null if unknown)

app/src/main/java/com/geeksville/mesh/database/PacketRepository.kt

@@ -3,9 +3,11 @@ package com.geeksville.mesh.database
 import androidx.lifecycle.LiveData
 import com.geeksville.mesh.database.dao.PacketDao
 import com.geeksville.mesh.database.entity.Packet
+import kotlinx.coroutines.flow.Flow
 
 class PacketRepository(private val packetDao : PacketDao) {
-    val allPackets : LiveData<List<Packet>> = packetDao.getAllPacket(500)
+    val allPackets : LiveData<List<Packet>> = packetDao.getAllPacket(MAX_ITEMS)
+    val allPacketsInReceiveOrder : Flow<List<Packet>> = packetDao.getAllPacketsInReceiveOrder(MAX_ITEMS)
 
     suspend fun insert(packet: Packet) {
         packetDao.insert(packet)
@@ -14,4 +16,9 @@ class PacketRepository(private val packetDao : PacketDao) {
     suspend fun deleteAll() {
         packetDao.deleteAll()
     }
+
+    companion object {
+        private const val MAX_ITEMS = 500
+    }
+
 }

app/src/main/java/com/geeksville/mesh/database/dao/PacketDao.kt

@@ -5,6 +5,7 @@ import androidx.room.Dao
 import androidx.room.Insert
 import androidx.room.Query
 import com.geeksville.mesh.database.entity.Packet
+import kotlinx.coroutines.flow.Flow
 
 @Dao
 interface PacketDao {
@@ -12,6 +13,9 @@ interface PacketDao {
     @Query("Select * from packet order by received_date desc limit 0,:maxItem")
     fun getAllPacket(maxItem: Int): LiveData<List<Packet>>
 
+    @Query("Select * from packet order by received_date asc limit 0,:maxItem")
+    fun getAllPacketsInReceiveOrder(maxItem: Int): Flow<List<Packet>>
+
     @Insert
     fun insert(packet: Packet)
 

app/src/main/java/com/geeksville/mesh/model/UIState.kt

@@ -12,15 +12,21 @@ import androidx.lifecycle.LiveData
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.viewModelScope
 import com.geeksville.android.Logging
-import com.geeksville.mesh.IMeshService
-import com.geeksville.mesh.MyNodeInfo
-import com.geeksville.mesh.RadioConfigProtos
+import com.geeksville.mesh.*
 import com.geeksville.mesh.database.MeshtasticDatabase
 import com.geeksville.mesh.database.PacketRepository
 import com.geeksville.mesh.database.entity.Packet
 import com.geeksville.mesh.service.MeshService
+import com.geeksville.mesh.ui.positionToMeter
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import java.io.BufferedWriter
+import java.io.FileWriter
+import java.text.SimpleDateFormat
+import java.util.*
+import kotlin.math.roundToInt
 
 /// Given a human name, strip out the first letter of the first three words and return that as the initials for
 /// that user. If the original name is only one word, strip vowels from the original name and if the result is
@@ -257,5 +263,96 @@ class UIViewModel(private val app: Application) : AndroidViewModel(app), Logging
                 errormsg("Can't set username on device, is device offline? ${ex.message}")
             }
     }
+
+    /**
+     * Write the persisted packet data out to a CSV file in the specified location.
+     */
+    fun saveMessagesCSV(file_uri: Uri) {
+        viewModelScope.launch(Dispatchers.Main) {
+            // Extract distances to this device from position messages and put (node,SNR,distance) in
+            // the file_uri
+            val myNodeNum = myNodeInfo.value?.myNodeNum ?: return@launch
+
+            // Capture the current node value while we're still on main thread
+            val nodes = nodeDB.nodes.value ?: emptyMap()
+
+            writeToUri(file_uri) { writer ->
+                // Create a map of nodes keyed by their ID
+                val nodesById = nodes.values.associateBy { it.num }
+
+                writer.appendLine("date,time,from,sender name,sender lat,sender long,rx lat,rx long,rx elevation,rx snr,distance,hop limit,payload")
+
+                // Packets are ordered by time, we keep most recent position of
+                // our device in localNodePosition.
+                var localNodePosition: MeshProtos.Position? = null
+                val dateFormat = SimpleDateFormat("yyyy-MM-dd,HH:mm:ss", Locale.getDefault())
+                repository.allPacketsInReceiveOrder.first().forEach { packet ->
+                    packet.proto?.let { proto ->
+                        packet.position?.let { position ->
+                            if (proto.from == myNodeNum) {
+                                localNodePosition = position
+                            } else {
+                                val rxDateTime = dateFormat.format(packet.received_date)
+                                val rxFrom = proto.from.toUInt()
+                                val senderName = nodesById[proto.from]?.user?.longName ?: ""
+
+                                // sender lat & long
+                                val senderPos = packet.position
+                                    ?.let { p -> Position(p) }
+                                    ?.takeIf { p -> p.isValid() }
+                                val senderLat = senderPos?.latitude ?: ""
+                                val senderLong = senderPos?.longitude ?: ""
+
+                                // rx lat, long, and elevation
+                                val rxPos = localNodePosition
+                                    ?.let { p -> Position(p) }
+                                    ?.takeIf { p -> p.isValid() }
+                                val rxLat = rxPos?.latitude ?: ""
+                                val rxLong = rxPos?.longitude ?: ""
+                                val rxAlt = rxPos?.altitude ?: ""
+                                val rxSnr = "%f".format(proto.rxSnr)
+
+                                // Calculate the distance if both positions are valid
+                                val dist = if (senderPos == null || rxPos == null) {
+                                    ""
+                                } else {
+                                    positionToMeter(
+                                        localNodePosition!!,
+                                        position
+                                    ).roundToInt().toString()
+                                }
+
+                                val hopLimit = proto.hopLimit
+
+                                val payload = when {
+                                    proto.decoded.portnumValue != Portnums.PortNum.TEXT_MESSAGE_APP_VALUE -> "<${proto.decoded.portnum}>"
+                                    proto.hasDecoded() -> "\"" + proto.decoded.payload.toStringUtf8()
+                                        .replace("\"", "\\\"") + "\""
+                                    proto.hasEncrypted() -> "${proto.encrypted.size()} encrypted bytes"
+                                    else -> ""
+                                }
+
+                                //  date,time,from,sender name,sender lat,sender long,rx lat,rx long,rx elevation,rx snr,distance,hop limit,payload
+                                writer.appendLine("$rxDateTime,$rxFrom,$senderName,$senderLat,$senderLong,$rxLat,$rxLong,$rxAlt,$rxSnr,$dist,$hopLimit,$payload")
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private suspend inline fun writeToUri(uri: Uri, crossinline block: suspend (BufferedWriter) -> Unit) {
+        withContext(Dispatchers.IO) {
+            app.contentResolver.openFileDescriptor(uri, "wt")?.use { parcelFileDescriptor ->
+                FileWriter(parcelFileDescriptor.fileDescriptor).use { fileWriter ->
+                    BufferedWriter(fileWriter).use { writer ->
+                        block.invoke(writer)
+                    }
+                }
+            }
+        }
+    }
+
 }