Fetching satellite data asynchronously, increased build speed

app/src/main/java/com/rtbishop/look4sat/framework/local/EntriesDao.kt

@@ -34,28 +34,24 @@ interface EntriesDao {
 
     @Transaction
     suspend fun updateEntries(entries: List<SatEntry>) {
-        val savedSelection = getSelection()
+        val entriesSelection = getEntriesSelection()
         insertEntries(entries)
-        restoreSelection(savedSelection, true)
+        restoreSelection(entriesSelection, true)
     }
 
     @Query("SELECT catnum FROM entries WHERE isSelected = 1")
-    suspend fun getSelection(): List<Int>
+    suspend fun getEntriesSelection(): List<Int>
 
     @Insert(onConflict = OnConflictStrategy.REPLACE)
     suspend fun insertEntries(entries: List<SatEntry>)
 
     @Transaction
     suspend fun restoreSelection(catnums: List<Int>, isSelected: Boolean) {
-        clearEntriesSelection()
+        updateEntriesSelection(catnums, isSelected)
-        updateSelection(catnums, isSelected)
     }
 
-    @Query("UPDATE entries SET isSelected = 0")
-    suspend fun clearEntriesSelection()
-
     @Transaction
-    suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean) {
+    suspend fun updateEntriesSelection(catnums: List<Int>, isSelected: Boolean) {
         catnums.forEach { catnum -> updateEntrySelection(catnum, isSelected) }
     }
 

app/src/main/java/com/rtbishop/look4sat/framework/local/LocalSource.kt

@@ -19,9 +19,12 @@ package com.rtbishop.look4sat.framework.local
 
 import com.rtbishop.look4sat.data.LocalDataSource
 import com.rtbishop.look4sat.domain.model.SatEntry
+import com.rtbishop.look4sat.domain.model.SatItem
 import com.rtbishop.look4sat.domain.model.Transmitter
+import com.rtbishop.look4sat.domain.predict.Satellite
 import com.rtbishop.look4sat.framework.*
 import com.rtbishop.look4sat.framework.model.Source
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
 
 class LocalSource(
@@ -30,31 +33,33 @@ class LocalSource(
     private val transmittersDao: TransmittersDao
 ) : LocalDataSource {
 
-    override fun getSatelliteItems() = entriesDao.getSatelliteItems().map { it.toDomainItems() }
+    override fun getSatelliteItems(): Flow<List<SatItem>> {
-
+        return entriesDao.getSatelliteItems().map { items -> items.toDomainItems() }
-    override suspend fun getSelectedSatellites() = entriesDao.getSelectedSatellites().map { entry ->
-        entry.tle.createSat()
     }
 
+    override suspend fun getSelectedSatellites(): List<Satellite> {
+        return entriesDao.getSelectedSatellites().map { entry -> entry.tle.createSat() }
+    }
+
+    override suspend fun getSources() = sourcesDao.getSources()
+
     override suspend fun getTransmitters(catnum: Int): List<Transmitter> {
         return transmittersDao.getTransmitters(catnum).toDomain()
     }
 
-    override suspend fun getWebSources() = sourcesDao.getSources()
-
     override suspend fun updateEntries(entries: List<SatEntry>) {
         entriesDao.updateEntries(entries.toFrameworkEntries())
     }
 
-    override suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean) {
+    override suspend fun updateEntriesSelection(catnums: List<Int>, isSelected: Boolean) {
-        entriesDao.updateSelection(catnums, isSelected)
+        entriesDao.updateEntriesSelection(catnums, isSelected)
+    }
+
+    override suspend fun updateSources(sources: List<String>) {
+        sourcesDao.updateSources(sources.map { sourceUrl -> Source(sourceUrl) })
     }
 
     override suspend fun updateTransmitters(transmitters: List<Transmitter>) {
         transmittersDao.updateTransmitters(transmitters.toFramework())
     }
-
-    override suspend fun updateWebSources(sources: List<String>) {
-        sourcesDao.updateSources(sources.map { Source(it) })
-    }
 }

app/src/main/java/com/rtbishop/look4sat/injection/AppModule.kt

@@ -28,27 +28,32 @@ import dagger.Provides
 import dagger.hilt.InstallIn
 import dagger.hilt.android.qualifiers.ApplicationContext
 import dagger.hilt.components.SingletonComponent
+import javax.inject.Singleton
 
 @Module
 @InstallIn(SingletonComponent::class)
 object AppModule {
 
     @Provides
+    @Singleton
     fun provideContentResolver(@ApplicationContext context: Context): ContentResolver {
         return context.contentResolver
     }
 
     @Provides
+    @Singleton
     fun provideLocationManager(@ApplicationContext context: Context): LocationManager {
         return context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
     }
 
     @Provides
+    @Singleton
     fun provideSensorManager(@ApplicationContext context: Context): SensorManager {
         return context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
     }
 
     @Provides
+    @Singleton
     fun provideSharedPreferences(@ApplicationContext context: Context): SharedPreferences {
         return PreferenceManager.getDefaultSharedPreferences(context)
     }

app/src/main/java/com/rtbishop/look4sat/injection/CoreModule.kt

@@ -50,7 +50,7 @@ object CoreModule {
             .addMigrations(MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4).build()
         val localSource = LocalSource(db.entriesDao(), db.sourcesDao(), db.transmittersDao())
         val remoteSource = RemoteSource(ioDispatcher)
-        return DefaultRepository(dataParser, localSource, remoteSource, ioDispatcher)
+        return DefaultRepository(dataParser, localSource, remoteSource)
     }
 
     @Provides

app/src/main/java/com/rtbishop/look4sat/presentation/Extensions.kt

@@ -17,10 +17,8 @@
  */
 package com.rtbishop.look4sat.presentation
 
-import android.net.Uri
 import android.os.Bundle
 import android.view.View
-import android.webkit.URLUtil
 import androidx.annotation.IdRes
 import androidx.fragment.app.Fragment
 import androidx.lifecycle.Lifecycle
@@ -31,9 +29,6 @@ import androidx.navigation.Navigator
 import androidx.navigation.fragment.findNavController
 import com.google.android.material.snackbar.Snackbar
 import com.rtbishop.look4sat.R
-import org.json.JSONArray
-import org.json.JSONException
-import org.json.JSONObject
 import java.security.MessageDigest
 import java.text.ParseException
 import java.text.SimpleDateFormat
@@ -104,25 +99,13 @@ fun Date.toString(format: String): String {
 }
 
 fun String.toDate(format: String): Date? {
-    val dateFormatter = SimpleDateFormat(format, Locale.US)
     return try {
-        dateFormatter.parse(this)
+        return SimpleDateFormat(format, Locale.US).parse(this)
     } catch (e: ParseException) {
         null
     }
 }
 
-fun String.toUri(): Uri? {
-    return try {
-        if (URLUtil.isValidUrl(this))
-            Uri.parse(this)
-        else
-            null
-    } catch (e: Exception) {
-        null
-    }
-}
-
 fun String.md5(): String {
     val bytes = MessageDigest.getInstance("MD5").digest(this.toByteArray())
     return bytes.joinToString("") { "%02x".format(it) }
@@ -139,76 +122,3 @@ fun String.isEmailValid(): Boolean {
     val matcher = pattern.matcher(this)
     return matcher.matches()
 }
-
-fun String.getJsonObjectOrNull(): JSONObject? {
-    return try {
-        JSONObject(this)
-    } catch (e: JSONException) {
-        null
-    }
-}
-
-fun String.getJsonArrayOrNull(): JSONArray? {
-    return try {
-        JSONArray(this)
-    } catch (e: JSONException) {
-        null
-    }
-}
-
-fun JSONObject.getIntOrNull(name: String): Int? =
-    try {
-        getInt(name)
-    } catch (e: JSONException) {
-        val strValue = getStringOrNull(name)
-        strValue?.toIntOrNull()
-    }
-
-fun JSONObject.getDoubleOrNull(name: String): Double? =
-    try {
-        getDouble(name)
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getLongOrNull(name: String): Long? =
-    try {
-        getLong(name)
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getStringOrNull(name: String): String? =
-    try {
-        getString(name).trim()
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getBooleanOrNull(name: String): Boolean? =
-    try {
-        getBoolean(name)
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getObjectOrNull(name: String): JSONObject? =
-    try {
-        getJSONObject(name)
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getArrayOrNull(name: String): JSONArray? =
-    try {
-        getJSONArray(name)
-    } catch (e: JSONException) {
-        null
-    }
-
-fun JSONObject.getArrayOrEmpty(name: String): JSONArray =
-    try {
-        getJSONArray(name)
-    } catch (e: JSONException) {
-        JSONArray()
-    }

app/src/main/res/drawable/ic_refresh.xml

@@ -5,5 +5,5 @@
     android:viewportHeight="24">
     <path
         android:fillColor="@color/themeLight"
-        android:pathData="M17.65,6.35C16.2,4.9 14.21,4 12,4c-4.42,0 -7.99,3.58 -7.99,8s3.57,8 7.99,8c3.73,0 6.84,-2.55 7.73,-6h-2.08c-0.82,2.33 -3.04,4 -5.65,4 -3.31,0 -6,-2.69 -6,-6s2.69,-6 6,-6c1.66,0 3.14,0.69 4.22,1.78L13,11h7V4l-2.35,2.35z" />
+        android:pathData="M19,8l-4,4h3c0,3.31 -2.69,6 -6,6 -1.01,0 -1.97,-0.25 -2.8,-0.7l-1.46,1.46C8.97,19.54 10.43,20 12,20c4.42,0 8,-3.58 8,-8h3l-4,-4zM6,12c0,-3.31 2.69,-6 6,-6 1.01,0 1.97,0.25 2.8,0.7l1.46,-1.46C15.03,4.46 13.57,4 12,4c-4.42,0 -8,3.58 -8,8H1l4,4 4,-4H6z" />
 </vector>

app/src/main/res/drawable/ic_update_file.xml

@@ -5,5 +5,5 @@
     android:viewportHeight="24">
     <path
         android:fillColor="@color/themeLight"
-        android:pathData="M18,2h-8L4.02,8 4,20c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2L20,4c0,-1.1 -0.9,-2 -2,-2zM12,8h-2L10,4h2v4zM15,8h-2L13,4h2v4zM18,8h-2L16,4h2v4z" />
+        android:pathData="M6,2c-1.1,0 -1.99,0.9 -1.99,2L4,20c0,1.1 0.89,2 1.99,2L18,22c1.1,0 2,-0.9 2,-2L20,8l-6,-6L6,2zM13,9L13,3.5L18.5,9L13,9z" />
 </vector>

app/src/main/res/drawable/ic_update_web.xml

@@ -5,5 +5,5 @@
     android:viewportHeight="24">
     <path
         android:fillColor="@color/themeLight"
-        android:pathData="M19,8l-4,4h3c0,3.31 -2.69,6 -6,6 -1.01,0 -1.97,-0.25 -2.8,-0.7l-1.46,1.46C8.97,19.54 10.43,20 12,20c4.42,0 8,-3.58 8,-8h3l-4,-4zM6,12c0,-3.31 2.69,-6 6,-6 1.01,0 1.97,0.25 2.8,0.7l1.46,-1.46C15.03,4.46 13.57,4 12,4c-4.42,0 -8,3.58 -8,8H1l4,4 4,-4H6z" />
+        android:pathData="M19.35,10.04C18.67,6.59 15.64,4 12,4 9.11,4 6.6,5.64 5.35,8.04 2.34,8.36 0,10.91 0,14c0,3.31 2.69,6 6,6h13c2.76,0 5,-2.24 5,-5 0,-2.64 -2.05,-4.78 -4.65,-4.96zM17,13l-5,5 -5,-5h3V9h4v4h3z" />
 </vector>

core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt

@@ -20,28 +20,28 @@ package com.rtbishop.look4sat.data
 import com.rtbishop.look4sat.domain.DataParser
 import com.rtbishop.look4sat.domain.DataRepository
 import com.rtbishop.look4sat.domain.model.SatEntry
-import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.withContext
 import java.io.InputStream
 import java.util.zip.ZipInputStream
+import kotlin.system.measureTimeMillis
 
 class DefaultRepository(
     private val dataParser: DataParser,
     private val localSource: LocalDataSource,
-    private val remoteSource: RemoteDataSource,
+    private val remoteSource: RemoteDataSource
-    private val repoDispatcher: CoroutineDispatcher
 ) : DataRepository {
 
     override val defaultSelection = listOf(43700, 25544, 25338, 28654, 33591, 40069, 27607, 24278)
     override val defaultSources = listOf(
-        "https://celestrak.com/NORAD/elements/active.txt",
+        "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=csv",
         "https://amsat.org/tle/current/nasabare.txt",
         "https://www.prismnet.com/~mmccants/tles/classfd.zip",
         "https://www.prismnet.com/~mmccants/tles/inttles.zip"
     )
-    override val transmittersSource = "https://db.satnogs.org/api/transmitters/"
+    override val transmittersSource = "https://db.satnogs.org/api/transmitters/?format=json"
 
     override fun getSatelliteItems() = localSource.getSatelliteItems()
 
@@ -49,36 +49,46 @@ class DefaultRepository(
 
     override suspend fun getTransmitters(catnum: Int) = localSource.getTransmitters(catnum)
 
-    override suspend fun getWebSources() = localSource.getWebSources().also { sources ->
+    override suspend fun getWebSources() = localSource.getSources().also { sources ->
         return if (sources.isNotEmpty()) sources
         else defaultSources
     }
 
-    override suspend fun updateDataFromFile(stream: InputStream) = withContext(repoDispatcher) {
+    override suspend fun updateDataFromFile(stream: InputStream) {
         localSource.updateEntries(importSatellites(stream))
     }
 
     override suspend fun updateDataFromWeb(sources: List<String>) {
         coroutineScope {
-            launch(repoDispatcher) {
+            launch {
-                localSource.updateWebSources(sources)
+                localSource.updateSources(sources)
             }
-            launch(repoDispatcher) {
+            launch {
-                val streams = mutableListOf<InputStream>()
+                val updateTimeMillis = measureTimeMillis {
-                val entries = mutableListOf<SatEntry>()
+                    val fetchesMap = mutableMapOf<String, Deferred<InputStream>>()
-                sources.forEach { source ->
+                    val streamsMap = mutableMapOf<String, InputStream>()
-                    val fileStream = remoteSource.fetchFileStream(source)
+                    val streams = mutableListOf<InputStream>()
-                    if (source.contains(".zip", true)) {
+                    val entries = mutableListOf<SatEntry>()
-                        val zipStream = ZipInputStream(fileStream).apply { nextEntry }
+                    sources.forEach { fetchesMap[it] = async { remoteSource.fetchFileStream(it) } }
-                        streams.add(zipStream)
+                    fetchesMap.forEach { streamsMap[it.key] = it.value.await() }
-                    } else {
+                    streamsMap.forEach { stream ->
-                        streams.add(fileStream)
+                        when {
+                            stream.key.contains("=csv", true) -> {
+                                val tles = dataParser.parseCSVStream(stream.value)
+                                entries.addAll(tles.map { tle -> SatEntry(tle) })
+                            }
+                            stream.key.contains(".zip", true) -> {
+                                streams.add(ZipInputStream(stream.value).apply { nextEntry })
+                            }
+                            else -> streams.add(stream.value)
+                        }
                     }
+                    streams.forEach { stream -> entries.addAll(importSatellites(stream)) }
+                    localSource.updateEntries(entries)
                 }
-                streams.forEach { stream -> entries.addAll(importSatellites(stream)) }
+                println("Update from web took $updateTimeMillis ms")
-                localSource.updateEntries(entries)
             }
-            launch(repoDispatcher) {
+            launch {
                 val jsonStream = remoteSource.fetchFileStream(transmittersSource)
                 val transmitters = dataParser.parseJSONStream(jsonStream)
                 localSource.updateTransmitters(transmitters)
@@ -87,7 +97,7 @@ class DefaultRepository(
     }
 
     override suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean) {
-        localSource.updateSelection(catnums, isSelected)
+        localSource.updateEntriesSelection(catnums, isSelected)
     }
 
     private suspend fun importSatellites(stream: InputStream): List<SatEntry> {

core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt

@@ -29,15 +29,15 @@ interface LocalDataSource {
 
     suspend fun getSelectedSatellites(): List<Satellite>
 
-    suspend fun getTransmitters(catnum: Int): List<Transmitter>
+    suspend fun getSources(): List<String>
 
-    suspend fun getWebSources(): List<String>
+    suspend fun getTransmitters(catnum: Int): List<Transmitter>
 
     suspend fun updateEntries(entries: List<SatEntry>)
 
-    suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean)
+    suspend fun updateEntriesSelection(catnums: List<Int>, isSelected: Boolean)
+
+    suspend fun updateSources(sources: List<String>)
 
     suspend fun updateTransmitters(transmitters: List<Transmitter>)
-
-    suspend fun updateWebSources(sources: List<String>)
 }

gradle.properties

@@ -19,4 +19,4 @@ android.useAndroidX=true
 android.enableJetifier=true
 # Kotlin code style for this project: "official" or "obsolete":
 kotlin.code.style=official
-kapt.use.worker.api=false
+kapt.use.worker.api=true