From 367ff7c75c6ea35d6e2d4dccfaa2233acb9f346a Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Tue, 18 Oct 2022 11:18:35 -0400 Subject: [PATCH] Always use CDSI. --- .../contacts/sync/ContactDiscovery.kt | 66 +--- .../sync/ContactDiscoveryRefreshV1.java | 337 ------------------ .../securesms/util/FeatureFlags.java | 42 +-- 3 files changed, 5 insertions(+), 440 deletions(-) delete mode 100644 app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscoveryRefreshV1.java diff --git a/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscovery.kt b/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscovery.kt index 17d848ce5..98c0c83ec 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscovery.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscovery.kt @@ -13,7 +13,6 @@ import org.signal.contacts.SystemContactsRepository.ContactIterator import org.signal.contacts.SystemContactsRepository.ContactPhoneDetails import org.signal.core.util.Stopwatch import org.signal.core.util.StringUtil -import org.signal.core.util.concurrent.SignalExecutors import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.BuildConfig import org.thoughtcrime.securesms.R @@ -38,9 +37,6 @@ import org.whispersystems.signalservice.api.push.SignalServiceAddress import org.whispersystems.signalservice.api.util.UuidUtil import java.io.IOException import java.util.Calendar -import java.util.concurrent.Callable -import java.util.concurrent.ExecutionException -import java.util.concurrent.Future /** * Methods for discovering which users are registered and marking them as such in the database. @@ -78,15 +74,7 @@ object ContactDiscovery { context = context, descriptor = "refresh-all", refresh = { - if (FeatureFlags.phoneNumberPrivacy()) { - ContactDiscoveryRefreshV2.refreshAll(context, useCompat = false, ignoreResults = false) - } else if (FeatureFlags.cdsV2Compat()) { - ContactDiscoveryRefreshV2.refreshAll(context, useCompat = true, ignoreResults = false) - } else if (FeatureFlags.cdsV2LoadTesting()) { - loadTestRefreshAll(context) - } else { - ContactDiscoveryRefreshV1.refreshAll(context) - } + ContactDiscoveryRefreshV2.refreshAll(context, useCompat = !FeatureFlags.phoneNumberPrivacy(), ignoreResults = false) }, removeSystemContactLinksIfMissing = true, notifyOfNewUsers = notifyOfNewUsers @@ -103,15 +91,7 @@ object ContactDiscovery { context = context, descriptor = "refresh-multiple", refresh = { - if (FeatureFlags.phoneNumberPrivacy()) { - ContactDiscoveryRefreshV2.refresh(context, recipients, useCompat = false, ignoreResults = false) - } else if (FeatureFlags.cdsV2Compat()) { - ContactDiscoveryRefreshV2.refresh(context, recipients, useCompat = true, ignoreResults = false) - } else if (FeatureFlags.cdsV2LoadTesting()) { - loadTestRefresh(context, recipients) - } else { - ContactDiscoveryRefreshV1.refresh(context, recipients) - } + ContactDiscoveryRefreshV2.refresh(context, recipients, useCompat = !FeatureFlags.phoneNumberPrivacy(), ignoreResults = false) }, removeSystemContactLinksIfMissing = false, notifyOfNewUsers = notifyOfNewUsers @@ -126,15 +106,7 @@ object ContactDiscovery { context = context, descriptor = "refresh-single", refresh = { - if (FeatureFlags.phoneNumberPrivacy()) { - ContactDiscoveryRefreshV2.refresh(context, listOf(recipient), useCompat = false, ignoreResults = false) - } else if (FeatureFlags.cdsV2Compat()) { - ContactDiscoveryRefreshV2.refresh(context, listOf(recipient), useCompat = true, ignoreResults = false) - } else if (FeatureFlags.cdsV2LoadTesting()) { - loadTestRefresh(context, listOf(recipient)) - } else { - ContactDiscoveryRefreshV1.refresh(context, listOf(recipient)) - } + ContactDiscoveryRefreshV2.refresh(context, listOf(recipient), useCompat = !FeatureFlags.phoneNumberPrivacy(), ignoreResults = false) }, removeSystemContactLinksIfMissing = false, notifyOfNewUsers = notifyOfNewUsers @@ -383,38 +355,6 @@ object ContactDiscovery { ApplicationDependencies.getProtocolStore().pni().containsSession(protocolAddress) } - private fun loadTestRefreshAll(context: Context): RefreshResult { - return loadTestOperation( - { ContactDiscoveryRefreshV1.refreshAll(context) }, - { ContactDiscoveryRefreshV2.refreshAll(context, useCompat = false, ignoreResults = true) } - ) - } - - private fun loadTestRefresh(context: Context, recipients: List): RefreshResult { - return loadTestOperation( - { ContactDiscoveryRefreshV1.refresh(context, recipients) }, - { ContactDiscoveryRefreshV2.refresh(context, recipients, useCompat = false, ignoreResults = true) } - ) - } - - private fun loadTestOperation(operationV1: Callable, operationV2: Callable): RefreshResult { - val v1Future: Future = SignalExecutors.UNBOUNDED.submit(operationV1) - val v2Future: Future = SignalExecutors.UNBOUNDED.submit(operationV2) - - try { - v2Future.get() - } catch (e: Throwable) { - Log.w(TAG, "Failed to complete the V2 fetch!", e) - } - - try { - return v1Future.get() - } catch (e: ExecutionException) { - Log.w(TAG, "Hit exception during V1 fetch!", e) - throw e.cause!! - } - } - class RefreshResult( val registeredIds: Set, val rewrites: Map diff --git a/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscoveryRefreshV1.java b/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscoveryRefreshV1.java deleted file mode 100644 index 49ffbadca..000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/contacts/sync/ContactDiscoveryRefreshV1.java +++ /dev/null @@ -1,337 +0,0 @@ -package org.thoughtcrime.securesms.contacts.sync; - -import android.content.Context; - -import androidx.annotation.NonNull; -import androidx.annotation.WorkerThread; - -import com.annimon.stream.Collectors; -import com.annimon.stream.Stream; - -import org.signal.contacts.SystemContactsRepository; -import org.signal.core.util.concurrent.RxExtensions; -import org.signal.core.util.logging.Log; -import org.signal.libsignal.protocol.InvalidKeyException; -import org.signal.libsignal.protocol.util.Pair; -import org.thoughtcrime.securesms.BuildConfig; -import org.thoughtcrime.securesms.contacts.sync.ContactDiscovery.RefreshResult; -import org.thoughtcrime.securesms.database.RecipientDatabase; -import org.thoughtcrime.securesms.database.SignalDatabase; -import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; -import org.thoughtcrime.securesms.jobs.MultiDeviceContactUpdateJob; -import org.thoughtcrime.securesms.jobs.RetrieveProfileJob; -import org.thoughtcrime.securesms.keyvalue.SignalStore; -import org.thoughtcrime.securesms.phonenumbers.PhoneNumberFormatter; -import org.thoughtcrime.securesms.push.IasTrustStore; -import org.thoughtcrime.securesms.recipients.Recipient; -import org.thoughtcrime.securesms.recipients.RecipientId; -import org.thoughtcrime.securesms.util.ProfileUtil; -import org.signal.core.util.SetUtil; -import org.signal.core.util.Stopwatch; -import org.thoughtcrime.securesms.util.TextSecurePreferences; -import org.whispersystems.signalservice.api.SignalServiceAccountManager; -import org.whispersystems.signalservice.api.profiles.ProfileAndCredential; -import org.whispersystems.signalservice.api.profiles.SignalServiceProfile; -import org.whispersystems.signalservice.api.push.ACI; -import org.whispersystems.signalservice.api.push.TrustStore; -import org.whispersystems.signalservice.api.services.ProfileService; -import org.whispersystems.signalservice.internal.ServiceResponse; -import org.whispersystems.signalservice.internal.contacts.crypto.Quote; -import org.whispersystems.signalservice.internal.contacts.crypto.UnauthenticatedQuoteException; -import org.whispersystems.signalservice.internal.contacts.crypto.UnauthenticatedResponseException; - -import java.io.IOException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.SignatureException; -import java.security.cert.CertificateException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.schedulers.Schedulers; - -/** - * Manages all the stuff around determining if a user is registered or not. - */ -class ContactDiscoveryRefreshV1 { - - // Using Log.tag will cut off the version number - private static final String TAG = "CdsRefreshV1"; - - private static final int MAX_NUMBERS = 20_500; - - @WorkerThread - static @NonNull RefreshResult refreshAll(@NonNull Context context) throws IOException { - RecipientDatabase recipientDatabase = SignalDatabase.recipients(); - Set databaseE164s = sanitizeNumbers(recipientDatabase.getAllE164s()); - Set systemE164s = sanitizeNumbers(Stream.of(SystemContactsRepository.getAllDisplayNumbers(context)) - .map(number -> PhoneNumberFormatter.get(context).format(number)) - .collect(Collectors.toSet())); - - return refreshNumbers(context, databaseE164s, systemE164s); - } - - @WorkerThread - static @NonNull RefreshResult refresh(@NonNull Context context, @NonNull List recipients) throws IOException { - RecipientDatabase recipientDatabase = SignalDatabase.recipients(); - - for (Recipient recipient : recipients) { - if (recipient.hasServiceId() && !recipient.hasE164()) { - if (ApplicationDependencies.getSignalServiceAccountManager().isIdentifierRegistered(recipient.requireServiceId())) { - recipientDatabase.markRegistered(recipient.getId(), recipient.requireServiceId()); - } else { - recipientDatabase.markUnregistered(recipient.getId()); - } - } - } - - Set numbers = Stream.of(recipients) - .filter(Recipient::hasE164) - .map(Recipient::requireE164) - .collect(Collectors.toSet()); - - if (numbers.size() < recipients.size()) { - Log.w(TAG, "We were asked to refresh " + recipients.size() + " numbers, but filtered that down to " + numbers.size()); - } - - return refreshNumbers(context, numbers, numbers); - } - - @WorkerThread - private static RefreshResult refreshNumbers(@NonNull Context context, @NonNull Set databaseNumbers, @NonNull Set systemNumbers) throws IOException { - RecipientDatabase recipientDatabase = SignalDatabase.recipients(); - Set allNumbers = SetUtil.union(databaseNumbers, systemNumbers); - - if (allNumbers.isEmpty()) { - Log.w(TAG, "No numbers to refresh!"); - return new RefreshResult(Collections.emptySet(), Collections.emptyMap()); - } - - Stopwatch stopwatch = new Stopwatch("refresh"); - - ContactIntersection result = getIntersection(context, databaseNumbers, systemNumbers); - - stopwatch.split("network"); - - if (result.getNumberRewrites().size() > 0) { - Log.i(TAG, "[getDirectoryResult] Need to rewrite some numbers."); - recipientDatabase.rewritePhoneNumbers(result.getNumberRewrites()); - } - - Map aciMap = recipientDatabase.bulkProcessCdsResult(result.getRegisteredNumbers()); - Set activeNumbers = result.getRegisteredNumbers().keySet(); - Set activeIds = aciMap.keySet(); - Set inactiveIds = Stream.of(allNumbers) - .filterNot(activeNumbers::contains) - .filterNot(n -> result.getNumberRewrites().containsKey(n)) - .filterNot(n -> result.getIgnoredNumbers().contains(n)) - .map(recipientDatabase::getOrInsertFromE164) - .collect(Collectors.toSet()); - - stopwatch.split("process-cds"); - - UnlistedResult unlistedResult = filterForUnlistedUsers(context, inactiveIds); - - inactiveIds.removeAll(unlistedResult.getPossiblyActive()); - - if (unlistedResult.getRetries().size() > 0) { - Log.i(TAG, "Some profile fetches failed to resolve. Assuming not-inactive for now and scheduling a retry."); - RetrieveProfileJob.enqueue(unlistedResult.getRetries()); - } - stopwatch.split("handle-unlisted"); - - recipientDatabase.bulkUpdatedRegisteredStatus(aciMap, inactiveIds); - stopwatch.split("update-registered"); - - - if (TextSecurePreferences.isMultiDevice(context)) { - ApplicationDependencies.getJobManager().add(new MultiDeviceContactUpdateJob()); - } - - stopwatch.stop(TAG); - - return new RefreshResult(activeIds, result.getNumberRewrites()); - } - - private static Set sanitizeNumbers(@NonNull Set numbers) { - return Stream.of(numbers).filter(number -> { - try { - return number.startsWith("+") && number.length() > 1 && number.charAt(1) != '0' && Long.parseLong(number.substring(1)) > 0; - } catch (NumberFormatException e) { - return false; - } - }).collect(Collectors.toSet()); - } - - /** - * Users can mark themselves as 'unlisted' in CDS, meaning that even if CDS says they're - * unregistered, they might actually be registered. We need to double-check users who we already - * have UUIDs for. Also, we only want to bother doing this for users we have conversations for, - * so we will also only check for users that have a thread. - */ - private static UnlistedResult filterForUnlistedUsers(@NonNull Context context, @NonNull Set inactiveIds) { - List possiblyUnlisted = Stream.of(inactiveIds) - .map(Recipient::resolved) - .filter(Recipient::isRegistered) - .filter(Recipient::hasServiceId) - .filter(ContactDiscoveryRefreshV1::hasCommunicatedWith) - .toList(); - - List>>> requests = Stream.of(possiblyUnlisted) - .map(r -> ProfileUtil.retrieveProfile(context, r, SignalServiceProfile.RequestType.PROFILE) - .toObservable() - .timeout(5, TimeUnit.SECONDS) - .onErrorReturn(t -> new Pair<>(r, ServiceResponse.forUnknownError(t)))) - .toList(); - - try { - return RxExtensions.safeBlockingGet( - Observable.mergeDelayError(requests) - .observeOn(Schedulers.io(), true) - .scan(new UnlistedResult.Builder(), (builder, pair) -> { - Recipient recipient = pair.first(); - ProfileService.ProfileResponseProcessor processor = new ProfileService.ProfileResponseProcessor(pair.second()); - if (processor.hasResult()) { - builder.potentiallyActiveIds.add(recipient.getId()); - } else if (processor.genericIoError() || !processor.notFound()) { - builder.retries.add(recipient.getId()); - builder.potentiallyActiveIds.add(recipient.getId()); - } - - return builder; - }) - .lastOrError() - .map(UnlistedResult.Builder::build) - ); - } catch (InterruptedException e) { - Log.i(TAG, "Filter for unlisted profile fetches interrupted, fetch via job instead"); - UnlistedResult.Builder builder = new UnlistedResult.Builder(); - for (Recipient recipient : possiblyUnlisted) { - builder.retries.add(recipient.getId()); - } - return builder.build(); - } - } - - private static boolean hasCommunicatedWith(@NonNull Recipient recipient) { - ACI localAci = SignalStore.account().requireAci(); - - return SignalDatabase.threads().hasThread(recipient.getId()) || (recipient.hasServiceId() && SignalDatabase.sessions().hasSessionFor(localAci, recipient.requireServiceId().toString())); - } - - /** - * Retrieves the contact intersection using the current production CDS. - */ - private static ContactIntersection getIntersection(@NonNull Context context, - @NonNull Set databaseNumbers, - @NonNull Set systemNumbers) - throws IOException - { - Set allNumbers = SetUtil.union(databaseNumbers, systemNumbers); - FuzzyPhoneNumberHelper.InputResult inputResult = FuzzyPhoneNumberHelper.generateInput(allNumbers, databaseNumbers); - Set sanitizedNumbers = sanitizeNumbers(inputResult.getNumbers()); - Set ignoredNumbers = new HashSet<>(); - - if (sanitizedNumbers.size() > MAX_NUMBERS) { - Set randomlySelected = randomlySelect(sanitizedNumbers, MAX_NUMBERS); - - ignoredNumbers = SetUtil.difference(sanitizedNumbers, randomlySelected); - sanitizedNumbers = randomlySelected; - } - - SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager(); - KeyStore iasKeyStore = getIasKeyStore(context); - - try { - Map results = accountManager.getRegisteredUsers(iasKeyStore, sanitizedNumbers, BuildConfig.CDS_MRENCLAVE); - FuzzyPhoneNumberHelper.OutputResult outputResult = FuzzyPhoneNumberHelper.generateOutput(results, inputResult); - - return new ContactIntersection(outputResult.getNumbers(), outputResult.getRewrites(), ignoredNumbers); - } catch (SignatureException | UnauthenticatedQuoteException | UnauthenticatedResponseException | Quote.InvalidQuoteFormatException | InvalidKeyException e) { - Log.w(TAG, "Attestation error.", e); - throw new IOException(e); - } - } - - private static @NonNull Set randomlySelect(@NonNull Set numbers, int max) { - List list = new ArrayList<>(numbers); - Collections.shuffle(list); - - return new HashSet<>(list.subList(0, max)); - } - - private static KeyStore getIasKeyStore(@NonNull Context context) { - try { - TrustStore contactTrustStore = new IasTrustStore(context); - - KeyStore keyStore = KeyStore.getInstance("BKS"); - keyStore.load(contactTrustStore.getKeyStoreInputStream(), contactTrustStore.getKeyStorePassword().toCharArray()); - - return keyStore; - } catch (KeyStoreException | CertificateException | IOException | NoSuchAlgorithmException e) { - throw new AssertionError(e); - } - } - - static class ContactIntersection { - private final Map registeredNumbers; - private final Map numberRewrites; - private final Set ignoredNumbers; - - ContactIntersection(@NonNull Map registeredNumbers, - @NonNull Map numberRewrites, - @NonNull Set ignoredNumbers) - { - this.registeredNumbers = registeredNumbers; - this.numberRewrites = numberRewrites; - this.ignoredNumbers = ignoredNumbers; - } - - - @NonNull Map getRegisteredNumbers() { - return registeredNumbers; - } - - @NonNull Map getNumberRewrites() { - return numberRewrites; - } - - @NonNull Set getIgnoredNumbers() { - return ignoredNumbers; - } - } - - private static class UnlistedResult { - private final Set possiblyActive; - private final Set retries; - - private UnlistedResult(@NonNull Set possiblyActive, @NonNull Set retries) { - this.possiblyActive = possiblyActive; - this.retries = retries; - } - - @NonNull Set getPossiblyActive() { - return possiblyActive; - } - - @NonNull Set getRetries() { - return retries; - } - - private static class Builder { - final Set potentiallyActiveIds = new HashSet<>(); - final Set retries = new HashSet<>(); - - @NonNull UnlistedResult build() { - return new UnlistedResult(potentiallyActiveIds, retries); - } - } - } -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java index ca9bf335a..8c6d56b6a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java @@ -99,20 +99,12 @@ public final class FeatureFlags { private static final String CAMERAX_MODEL_BLOCKLIST = "android.cameraXModelBlockList"; private static final String CAMERAX_MIXED_MODEL_BLOCKLIST = "android.cameraXMixedModelBlockList"; private static final String RECIPIENT_MERGE_V2 = "android.recipientMergeV2"; - private static final String CDS_V2_LOAD_TEST = "android.cdsV2LoadTest"; private static final String SMS_EXPORTER = "android.sms.exporter.2"; - private static final String CDS_V2_COMPAT = "android.cdsV2Compat.4"; public static final String STORIES_LOCALE = "android.stories.locale.1"; private static final String HIDE_CONTACTS = "android.hide.contacts"; -<<<<<<< HEAD private static final String MEDIA_PREVIEW_V2 = "android.mediaPreviewV2"; private static final String SMS_EXPORT_MEGAPHONE_DELAY_DAYS = "android.smsExport.megaphoneDelayDays"; -||||||| parent of 90b7447a79 (Credit card validator implementations and spec tests.) - public static final String MEDIA_PREVIEW_V2 = "android.mediaPreviewV2"; -======= - public static final String MEDIA_PREVIEW_V2 = "android.mediaPreviewV2"; public static final String CREDIT_CARD_PAYMENTS = "android.credit.card.payments"; ->>>>>>> 90b7447a79 (Credit card validator implementations and spec tests.) /** * We will only store remote values for flags in this set. If you want a flag to be controllable @@ -164,20 +156,12 @@ public final class FeatureFlags { CAMERAX_MODEL_BLOCKLIST, CAMERAX_MIXED_MODEL_BLOCKLIST, RECIPIENT_MERGE_V2, - CDS_V2_LOAD_TEST, SMS_EXPORTER, - CDS_V2_COMPAT, STORIES_LOCALE, HIDE_CONTACTS, -<<<<<<< HEAD - MEDIA_PREVIEW_V2, - SMS_EXPORT_MEGAPHONE_DELAY_DAYS -||||||| parent of 90b7447a79 (Credit card validator implementations and spec tests.) - MEDIA_PREVIEW_V2 -======= MEDIA_PREVIEW_V2, + SMS_EXPORT_MEGAPHONE_DELAY_DAYS, CREDIT_CARD_PAYMENTS ->>>>>>> 90b7447a79 (Credit card validator implementations and spec tests.) ); @VisibleForTesting @@ -238,18 +222,10 @@ public final class FeatureFlags { TELECOM_MODEL_BLOCKLIST, CAMERAX_MODEL_BLOCKLIST, RECIPIENT_MERGE_V2, - CDS_V2_LOAD_TEST, - CDS_V2_COMPAT, STORIES, -<<<<<<< HEAD - MEDIA_PREVIEW_V2, - SMS_EXPORT_MEGAPHONE_DELAY_DAYS -||||||| parent of 90b7447a79 (Credit card validator implementations and spec tests.) - MEDIA_PREVIEW_V2 -======= + SMS_EXPORT_MEGAPHONE_DELAY_DAYS, MEDIA_PREVIEW_V2, CREDIT_CARD_PAYMENTS ->>>>>>> 90b7447a79 (Credit card validator implementations and spec tests.) ); /** @@ -568,13 +544,6 @@ public final class FeatureFlags { return getBoolean(RECIPIENT_MERGE_V2, false); } - /** - * Whether or not we should also query CDSv2 as a form of load test. - */ - public static boolean cdsV2LoadTesting() { - return getBoolean(CDS_V2_LOAD_TEST, false); - } - /** * Whether or not we should enable the SMS exporter * @@ -585,13 +554,6 @@ public final class FeatureFlags { return getBoolean(SMS_EXPORTER, false); } - /** - * Whether or not we should use CDSv2 with the compat flag on as our primary CDS. - */ - public static boolean cdsV2Compat() { - return getBoolean(CDS_V2_COMPAT, false); - } - /** * Whether or not users can hide contacts. *