From 9b76d427072f1cb7d5c3ef5c35caf285a89863da Mon Sep 17 00:00:00 2001 From: Arty Bishop Date: Sun, 24 Oct 2021 12:53:19 +0100 Subject: [PATCH] Fetching satellite data asynchronously, increased build speed --- .../look4sat/framework/local/EntriesDao.kt | 14 +-- .../look4sat/framework/local/LocalSource.kt | 29 +++--- .../rtbishop/look4sat/injection/AppModule.kt | 5 + .../rtbishop/look4sat/injection/CoreModule.kt | 2 +- .../look4sat/presentation/Extensions.kt | 92 +------------------ app/src/main/res/drawable/ic_refresh.xml | 2 +- app/src/main/res/drawable/ic_update_file.xml | 2 +- app/src/main/res/drawable/ic_update_web.xml | 2 +- .../look4sat/data/DefaultRepository.kt | 58 +++++++----- .../rtbishop/look4sat/data/LocalDataSource.kt | 10 +- gradle.properties | 2 +- gradlew | 0 12 files changed, 72 insertions(+), 146 deletions(-) mode change 100644 => 100755 gradlew diff --git a/app/src/main/java/com/rtbishop/look4sat/framework/local/EntriesDao.kt b/app/src/main/java/com/rtbishop/look4sat/framework/local/EntriesDao.kt index 4b2529a9..a926b821 100644 --- a/app/src/main/java/com/rtbishop/look4sat/framework/local/EntriesDao.kt +++ b/app/src/main/java/com/rtbishop/look4sat/framework/local/EntriesDao.kt @@ -34,28 +34,24 @@ interface EntriesDao { @Transaction suspend fun updateEntries(entries: List) { - val savedSelection = getSelection() + val entriesSelection = getEntriesSelection() insertEntries(entries) - restoreSelection(savedSelection, true) + restoreSelection(entriesSelection, true) } @Query("SELECT catnum FROM entries WHERE isSelected = 1") - suspend fun getSelection(): List + suspend fun getEntriesSelection(): List @Insert(onConflict = OnConflictStrategy.REPLACE) suspend fun insertEntries(entries: List) @Transaction suspend fun restoreSelection(catnums: List, isSelected: Boolean) { - clearEntriesSelection() - updateSelection(catnums, isSelected) + updateEntriesSelection(catnums, isSelected) } - @Query("UPDATE entries SET isSelected = 0") - suspend fun clearEntriesSelection() - @Transaction - suspend fun updateSelection(catnums: List, isSelected: Boolean) { + suspend fun updateEntriesSelection(catnums: List, isSelected: Boolean) { catnums.forEach { catnum -> updateEntrySelection(catnum, isSelected) } } diff --git a/app/src/main/java/com/rtbishop/look4sat/framework/local/LocalSource.kt b/app/src/main/java/com/rtbishop/look4sat/framework/local/LocalSource.kt index c89c11cb..90877f18 100644 --- a/app/src/main/java/com/rtbishop/look4sat/framework/local/LocalSource.kt +++ b/app/src/main/java/com/rtbishop/look4sat/framework/local/LocalSource.kt @@ -19,9 +19,12 @@ package com.rtbishop.look4sat.framework.local import com.rtbishop.look4sat.data.LocalDataSource import com.rtbishop.look4sat.domain.model.SatEntry +import com.rtbishop.look4sat.domain.model.SatItem import com.rtbishop.look4sat.domain.model.Transmitter +import com.rtbishop.look4sat.domain.predict.Satellite import com.rtbishop.look4sat.framework.* import com.rtbishop.look4sat.framework.model.Source +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map class LocalSource( @@ -30,31 +33,33 @@ class LocalSource( private val transmittersDao: TransmittersDao ) : LocalDataSource { - override fun getSatelliteItems() = entriesDao.getSatelliteItems().map { it.toDomainItems() } - - override suspend fun getSelectedSatellites() = entriesDao.getSelectedSatellites().map { entry -> - entry.tle.createSat() + override fun getSatelliteItems(): Flow> { + return entriesDao.getSatelliteItems().map { items -> items.toDomainItems() } } + override suspend fun getSelectedSatellites(): List { + return entriesDao.getSelectedSatellites().map { entry -> entry.tle.createSat() } + } + + override suspend fun getSources() = sourcesDao.getSources() + override suspend fun getTransmitters(catnum: Int): List { return transmittersDao.getTransmitters(catnum).toDomain() } - override suspend fun getWebSources() = sourcesDao.getSources() - override suspend fun updateEntries(entries: List) { entriesDao.updateEntries(entries.toFrameworkEntries()) } - override suspend fun updateSelection(catnums: List, isSelected: Boolean) { - entriesDao.updateSelection(catnums, isSelected) + override suspend fun updateEntriesSelection(catnums: List, isSelected: Boolean) { + entriesDao.updateEntriesSelection(catnums, isSelected) + } + + override suspend fun updateSources(sources: List) { + sourcesDao.updateSources(sources.map { sourceUrl -> Source(sourceUrl) }) } override suspend fun updateTransmitters(transmitters: List) { transmittersDao.updateTransmitters(transmitters.toFramework()) } - - override suspend fun updateWebSources(sources: List) { - sourcesDao.updateSources(sources.map { Source(it) }) - } } diff --git a/app/src/main/java/com/rtbishop/look4sat/injection/AppModule.kt b/app/src/main/java/com/rtbishop/look4sat/injection/AppModule.kt index 9d65dd0a..0c2b4f40 100644 --- a/app/src/main/java/com/rtbishop/look4sat/injection/AppModule.kt +++ b/app/src/main/java/com/rtbishop/look4sat/injection/AppModule.kt @@ -28,27 +28,32 @@ import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent +import javax.inject.Singleton @Module @InstallIn(SingletonComponent::class) object AppModule { @Provides + @Singleton fun provideContentResolver(@ApplicationContext context: Context): ContentResolver { return context.contentResolver } @Provides + @Singleton fun provideLocationManager(@ApplicationContext context: Context): LocationManager { return context.getSystemService(Context.LOCATION_SERVICE) as LocationManager } @Provides + @Singleton fun provideSensorManager(@ApplicationContext context: Context): SensorManager { return context.getSystemService(Context.SENSOR_SERVICE) as SensorManager } @Provides + @Singleton fun provideSharedPreferences(@ApplicationContext context: Context): SharedPreferences { return PreferenceManager.getDefaultSharedPreferences(context) } diff --git a/app/src/main/java/com/rtbishop/look4sat/injection/CoreModule.kt b/app/src/main/java/com/rtbishop/look4sat/injection/CoreModule.kt index b8c31d7f..a64e5e51 100644 --- a/app/src/main/java/com/rtbishop/look4sat/injection/CoreModule.kt +++ b/app/src/main/java/com/rtbishop/look4sat/injection/CoreModule.kt @@ -50,7 +50,7 @@ object CoreModule { .addMigrations(MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4).build() val localSource = LocalSource(db.entriesDao(), db.sourcesDao(), db.transmittersDao()) val remoteSource = RemoteSource(ioDispatcher) - return DefaultRepository(dataParser, localSource, remoteSource, ioDispatcher) + return DefaultRepository(dataParser, localSource, remoteSource) } @Provides diff --git a/app/src/main/java/com/rtbishop/look4sat/presentation/Extensions.kt b/app/src/main/java/com/rtbishop/look4sat/presentation/Extensions.kt index 1ab2a953..5584e576 100644 --- a/app/src/main/java/com/rtbishop/look4sat/presentation/Extensions.kt +++ b/app/src/main/java/com/rtbishop/look4sat/presentation/Extensions.kt @@ -17,10 +17,8 @@ */ package com.rtbishop.look4sat.presentation -import android.net.Uri import android.os.Bundle import android.view.View -import android.webkit.URLUtil import androidx.annotation.IdRes import androidx.fragment.app.Fragment import androidx.lifecycle.Lifecycle @@ -31,9 +29,6 @@ import androidx.navigation.Navigator import androidx.navigation.fragment.findNavController import com.google.android.material.snackbar.Snackbar import com.rtbishop.look4sat.R -import org.json.JSONArray -import org.json.JSONException -import org.json.JSONObject import java.security.MessageDigest import java.text.ParseException import java.text.SimpleDateFormat @@ -104,25 +99,13 @@ fun Date.toString(format: String): String { } fun String.toDate(format: String): Date? { - val dateFormatter = SimpleDateFormat(format, Locale.US) return try { - dateFormatter.parse(this) + return SimpleDateFormat(format, Locale.US).parse(this) } catch (e: ParseException) { null } } -fun String.toUri(): Uri? { - return try { - if (URLUtil.isValidUrl(this)) - Uri.parse(this) - else - null - } catch (e: Exception) { - null - } -} - fun String.md5(): String { val bytes = MessageDigest.getInstance("MD5").digest(this.toByteArray()) return bytes.joinToString("") { "%02x".format(it) } @@ -139,76 +122,3 @@ fun String.isEmailValid(): Boolean { val matcher = pattern.matcher(this) return matcher.matches() } - -fun String.getJsonObjectOrNull(): JSONObject? { - return try { - JSONObject(this) - } catch (e: JSONException) { - null - } -} - -fun String.getJsonArrayOrNull(): JSONArray? { - return try { - JSONArray(this) - } catch (e: JSONException) { - null - } -} - -fun JSONObject.getIntOrNull(name: String): Int? = - try { - getInt(name) - } catch (e: JSONException) { - val strValue = getStringOrNull(name) - strValue?.toIntOrNull() - } - -fun JSONObject.getDoubleOrNull(name: String): Double? = - try { - getDouble(name) - } catch (e: JSONException) { - null - } - -fun JSONObject.getLongOrNull(name: String): Long? = - try { - getLong(name) - } catch (e: JSONException) { - null - } - -fun JSONObject.getStringOrNull(name: String): String? = - try { - getString(name).trim() - } catch (e: JSONException) { - null - } - -fun JSONObject.getBooleanOrNull(name: String): Boolean? = - try { - getBoolean(name) - } catch (e: JSONException) { - null - } - -fun JSONObject.getObjectOrNull(name: String): JSONObject? = - try { - getJSONObject(name) - } catch (e: JSONException) { - null - } - -fun JSONObject.getArrayOrNull(name: String): JSONArray? = - try { - getJSONArray(name) - } catch (e: JSONException) { - null - } - -fun JSONObject.getArrayOrEmpty(name: String): JSONArray = - try { - getJSONArray(name) - } catch (e: JSONException) { - JSONArray() - } diff --git a/app/src/main/res/drawable/ic_refresh.xml b/app/src/main/res/drawable/ic_refresh.xml index 270203ce..a96341a9 100644 --- a/app/src/main/res/drawable/ic_refresh.xml +++ b/app/src/main/res/drawable/ic_refresh.xml @@ -5,5 +5,5 @@ android:viewportHeight="24"> + android:pathData="M19,8l-4,4h3c0,3.31 -2.69,6 -6,6 -1.01,0 -1.97,-0.25 -2.8,-0.7l-1.46,1.46C8.97,19.54 10.43,20 12,20c4.42,0 8,-3.58 8,-8h3l-4,-4zM6,12c0,-3.31 2.69,-6 6,-6 1.01,0 1.97,0.25 2.8,0.7l1.46,-1.46C15.03,4.46 13.57,4 12,4c-4.42,0 -8,3.58 -8,8H1l4,4 4,-4H6z" /> diff --git a/app/src/main/res/drawable/ic_update_file.xml b/app/src/main/res/drawable/ic_update_file.xml index 6724b2a8..acbb87a2 100644 --- a/app/src/main/res/drawable/ic_update_file.xml +++ b/app/src/main/res/drawable/ic_update_file.xml @@ -5,5 +5,5 @@ android:viewportHeight="24"> + android:pathData="M6,2c-1.1,0 -1.99,0.9 -1.99,2L4,20c0,1.1 0.89,2 1.99,2L18,22c1.1,0 2,-0.9 2,-2L20,8l-6,-6L6,2zM13,9L13,3.5L18.5,9L13,9z" /> diff --git a/app/src/main/res/drawable/ic_update_web.xml b/app/src/main/res/drawable/ic_update_web.xml index a96341a9..4788f844 100644 --- a/app/src/main/res/drawable/ic_update_web.xml +++ b/app/src/main/res/drawable/ic_update_web.xml @@ -5,5 +5,5 @@ android:viewportHeight="24"> + android:pathData="M19.35,10.04C18.67,6.59 15.64,4 12,4 9.11,4 6.6,5.64 5.35,8.04 2.34,8.36 0,10.91 0,14c0,3.31 2.69,6 6,6h13c2.76,0 5,-2.24 5,-5 0,-2.64 -2.05,-4.78 -4.65,-4.96zM17,13l-5,5 -5,-5h3V9h4v4h3z" /> diff --git a/core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt b/core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt index 8b7ffa76..592577d5 100644 --- a/core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt +++ b/core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt @@ -20,28 +20,28 @@ package com.rtbishop.look4sat.data import com.rtbishop.look4sat.domain.DataParser import com.rtbishop.look4sat.domain.DataRepository import com.rtbishop.look4sat.domain.model.SatEntry -import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext import java.io.InputStream import java.util.zip.ZipInputStream +import kotlin.system.measureTimeMillis class DefaultRepository( private val dataParser: DataParser, private val localSource: LocalDataSource, - private val remoteSource: RemoteDataSource, - private val repoDispatcher: CoroutineDispatcher + private val remoteSource: RemoteDataSource ) : DataRepository { override val defaultSelection = listOf(43700, 25544, 25338, 28654, 33591, 40069, 27607, 24278) override val defaultSources = listOf( - "https://celestrak.com/NORAD/elements/active.txt", + "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=csv", "https://amsat.org/tle/current/nasabare.txt", "https://www.prismnet.com/~mmccants/tles/classfd.zip", "https://www.prismnet.com/~mmccants/tles/inttles.zip" ) - override val transmittersSource = "https://db.satnogs.org/api/transmitters/" + override val transmittersSource = "https://db.satnogs.org/api/transmitters/?format=json" override fun getSatelliteItems() = localSource.getSatelliteItems() @@ -49,36 +49,46 @@ class DefaultRepository( override suspend fun getTransmitters(catnum: Int) = localSource.getTransmitters(catnum) - override suspend fun getWebSources() = localSource.getWebSources().also { sources -> + override suspend fun getWebSources() = localSource.getSources().also { sources -> return if (sources.isNotEmpty()) sources else defaultSources } - override suspend fun updateDataFromFile(stream: InputStream) = withContext(repoDispatcher) { + override suspend fun updateDataFromFile(stream: InputStream) { localSource.updateEntries(importSatellites(stream)) } override suspend fun updateDataFromWeb(sources: List) { coroutineScope { - launch(repoDispatcher) { - localSource.updateWebSources(sources) + launch { + localSource.updateSources(sources) } - launch(repoDispatcher) { - val streams = mutableListOf() - val entries = mutableListOf() - sources.forEach { source -> - val fileStream = remoteSource.fetchFileStream(source) - if (source.contains(".zip", true)) { - val zipStream = ZipInputStream(fileStream).apply { nextEntry } - streams.add(zipStream) - } else { - streams.add(fileStream) + launch { + val updateTimeMillis = measureTimeMillis { + val fetchesMap = mutableMapOf>() + val streamsMap = mutableMapOf() + val streams = mutableListOf() + val entries = mutableListOf() + sources.forEach { fetchesMap[it] = async { remoteSource.fetchFileStream(it) } } + fetchesMap.forEach { streamsMap[it.key] = it.value.await() } + streamsMap.forEach { stream -> + when { + stream.key.contains("=csv", true) -> { + val tles = dataParser.parseCSVStream(stream.value) + entries.addAll(tles.map { tle -> SatEntry(tle) }) + } + stream.key.contains(".zip", true) -> { + streams.add(ZipInputStream(stream.value).apply { nextEntry }) + } + else -> streams.add(stream.value) + } } + streams.forEach { stream -> entries.addAll(importSatellites(stream)) } + localSource.updateEntries(entries) } - streams.forEach { stream -> entries.addAll(importSatellites(stream)) } - localSource.updateEntries(entries) + println("Update from web took $updateTimeMillis ms") } - launch(repoDispatcher) { + launch { val jsonStream = remoteSource.fetchFileStream(transmittersSource) val transmitters = dataParser.parseJSONStream(jsonStream) localSource.updateTransmitters(transmitters) @@ -87,7 +97,7 @@ class DefaultRepository( } override suspend fun updateSelection(catnums: List, isSelected: Boolean) { - localSource.updateSelection(catnums, isSelected) + localSource.updateEntriesSelection(catnums, isSelected) } private suspend fun importSatellites(stream: InputStream): List { diff --git a/core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt b/core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt index c42564c3..eb003a36 100644 --- a/core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt +++ b/core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt @@ -29,15 +29,15 @@ interface LocalDataSource { suspend fun getSelectedSatellites(): List - suspend fun getTransmitters(catnum: Int): List + suspend fun getSources(): List - suspend fun getWebSources(): List + suspend fun getTransmitters(catnum: Int): List suspend fun updateEntries(entries: List) - suspend fun updateSelection(catnums: List, isSelected: Boolean) + suspend fun updateEntriesSelection(catnums: List, isSelected: Boolean) + + suspend fun updateSources(sources: List) suspend fun updateTransmitters(transmitters: List) - - suspend fun updateWebSources(sources: List) } diff --git a/gradle.properties b/gradle.properties index f6e3790d..462df245 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,4 +19,4 @@ android.useAndroidX=true android.enableJetifier=true # Kotlin code style for this project: "official" or "obsolete": kotlin.code.style=official -kapt.use.worker.api=false +kapt.use.worker.api=true diff --git a/gradlew b/gradlew old mode 100644 new mode 100755