Have a much longer backoff maximum for 5xx errors.

app/src/main/java/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java

@@ -263,13 +263,6 @@ public final class PushGroupSendJob extends PushSendJob {
     }
   }
 
-  @Override
-  public boolean onShouldRetry(@NonNull Exception exception) {
-    if (exception instanceof IOException)         return true;
-    if (exception instanceof RetryLaterException) return true;
-    return false;
-  }
-
   @Override
   public void onFailure() {
     DatabaseFactory.getMmsDatabase(context).markAsSentFailed(messageId);

app/src/main/java/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java

@@ -105,8 +105,7 @@ public class PushMediaSendJob extends PushSendJob {
 
   @Override
   public void onPushSend()
-      throws RetryLaterException, MmsException, NoSuchMessageException,
-             UndeliverableMessageException
+      throws IOException, MmsException, NoSuchMessageException, UndeliverableMessageException
   {
     ExpiringMessageManager expirationManager = ApplicationContext.getInstance(context).getExpiringMessageManager();
     MessageDatabase        database          = DatabaseFactory.getMmsDatabase(context);
@@ -175,12 +174,6 @@ public class PushMediaSendJob extends PushSendJob {
     }
   }
 
-  @Override
-  public boolean onShouldRetry(@NonNull Exception exception) {
-    if (exception instanceof RetryLaterException) return true;
-    return false;
-  }
-
   @Override
   public void onFailure() {
     DatabaseFactory.getMmsDatabase(context).markAsSentFailed(messageId);
@@ -188,8 +181,7 @@ public class PushMediaSendJob extends PushSendJob {
   }
 
   private boolean deliver(OutgoingMediaMessage message)
-      throws RetryLaterException, InsecureFallbackApprovalException, UntrustedIdentityException,
-             UndeliverableMessageException
+      throws IOException, InsecureFallbackApprovalException, UntrustedIdentityException, UndeliverableMessageException
   {
     if (message.getRecipient() == null) {
       throw new UndeliverableMessageException("No destination address.");
@@ -239,9 +231,6 @@ public class PushMediaSendJob extends PushSendJob {
       throw new UndeliverableMessageException(e);
     } catch (ServerRejectedException e) {
       throw new UndeliverableMessageException(e);
-    } catch (IOException e) {
-      warn(TAG, String.valueOf(message.getSentTimeMillis()), e);
-      throw new RetryLaterException(e);
     }
   }
 

app/src/main/java/org/thoughtcrime/securesms/jobs/PushSendJob.java

@@ -28,6 +28,7 @@ import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.events.PartProgressEvent;
 import org.thoughtcrime.securesms.jobmanager.Job;
 import org.thoughtcrime.securesms.jobmanager.JobManager;
+import org.thoughtcrime.securesms.jobmanager.impl.BackoffUtil;
 import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint;
 import org.thoughtcrime.securesms.keyvalue.CertificateType;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
@@ -38,9 +39,11 @@ import org.thoughtcrime.securesms.mms.PartAuthority;
 import org.thoughtcrime.securesms.recipients.Recipient;
 import org.thoughtcrime.securesms.recipients.RecipientId;
 import org.thoughtcrime.securesms.recipients.RecipientUtil;
+import org.thoughtcrime.securesms.transport.RetryLaterException;
 import org.thoughtcrime.securesms.util.Base64;
 import org.thoughtcrime.securesms.util.BitmapDecodingException;
 import org.thoughtcrime.securesms.util.BitmapUtil;
+import org.thoughtcrime.securesms.util.FeatureFlags;
 import org.thoughtcrime.securesms.util.Hex;
 import org.thoughtcrime.securesms.util.MediaUtil;
 import org.thoughtcrime.securesms.util.TextSecurePreferences;
@@ -56,6 +59,8 @@ import org.whispersystems.signalservice.api.messages.multidevice.SentTranscriptM
 import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage;
 import org.whispersystems.signalservice.api.messages.shared.SharedContact;
 import org.whispersystems.signalservice.api.push.SignalServiceAddress;
+import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException;
+import org.whispersystems.signalservice.api.push.exceptions.ServerRejectedException;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -113,6 +118,27 @@ public abstract class PushSendJob extends SendJob {
     return true;
   }
 
+  @Override
+  public boolean onShouldRetry(@NonNull Exception exception) {
+    if (exception instanceof ServerRejectedException) {
+      return false;
+    }
+
+    return exception instanceof IOException ||
+           exception instanceof RetryLaterException;
+  }
+
+  @Override
+  public long getNextRunAttemptBackoff(int pastAttemptCount, @NonNull Exception exception) {
+    if (exception instanceof NonSuccessfulResponseCodeException) {
+      if (((NonSuccessfulResponseCodeException) exception).is5xx()) {
+        return BackoffUtil.exponentialBackoff(pastAttemptCount, FeatureFlags.getServerErrorMaxBackoff());
+      }
+    }
+
+    return super.getNextRunAttemptBackoff(pastAttemptCount, exception);
+  }
+
   protected Optional<byte[]> getProfileKey(@NonNull Recipient recipient) {
     if (!recipient.resolve().isSystemContact() && !recipient.resolve().isProfileSharing()) {
       return Optional.absent();

app/src/main/java/org/thoughtcrime/securesms/jobs/PushTextSendJob.java

@@ -13,6 +13,7 @@ import org.thoughtcrime.securesms.database.model.SmsMessageRecord;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.jobmanager.Data;
 import org.thoughtcrime.securesms.jobmanager.Job;
+import org.thoughtcrime.securesms.jobmanager.impl.BackoffUtil;
 import org.thoughtcrime.securesms.recipients.Recipient;
 import org.thoughtcrime.securesms.recipients.RecipientId;
 import org.thoughtcrime.securesms.recipients.RecipientUtil;
@@ -29,10 +30,12 @@ import org.whispersystems.signalservice.api.crypto.UntrustedIdentityException;
 import org.whispersystems.signalservice.api.messages.SignalServiceDataMessage;
 import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage;
 import org.whispersystems.signalservice.api.push.SignalServiceAddress;
+import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException;
 import org.whispersystems.signalservice.api.push.exceptions.ServerRejectedException;
 import org.whispersystems.signalservice.api.push.exceptions.UnregisteredUserException;
 
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 
 public class PushTextSendJob extends PushSendJob {
 
@@ -69,7 +72,7 @@ public class PushTextSendJob extends PushSendJob {
   }
 
   @Override
-  public void onPushSend() throws NoSuchMessageException, RetryLaterException, UndeliverableMessageException {
+  public void onPushSend() throws IOException, NoSuchMessageException, UndeliverableMessageException {
     ExpiringMessageManager expirationManager = ApplicationContext.getInstance(context).getExpiringMessageManager();
     MessageDatabase        database          = DatabaseFactory.getSmsDatabase(context);
     SmsMessageRecord       record            = database.getSmsMessage(messageId);
@@ -132,13 +135,6 @@ public class PushTextSendJob extends PushSendJob {
     }
   }
 
-  @Override
-  public boolean onShouldRetry(@NonNull Exception exception) {
-    if (exception instanceof RetryLaterException) return true;
-
-    return false;
-  }
-
   @Override
   public void onFailure() {
     DatabaseFactory.getSmsDatabase(context).markAsSentFailed(messageId);
@@ -152,7 +148,7 @@ public class PushTextSendJob extends PushSendJob {
   }
 
   private boolean deliver(SmsMessageRecord message)
-      throws UntrustedIdentityException, InsecureFallbackApprovalException, RetryLaterException, UndeliverableMessageException
+      throws UntrustedIdentityException, InsecureFallbackApprovalException, UndeliverableMessageException, IOException
   {
     try {
       rotateSenderCertificateIfNecessary();
@@ -187,9 +183,6 @@ public class PushTextSendJob extends PushSendJob {
       throw new InsecureFallbackApprovalException(e);
     } catch (ServerRejectedException e) {
       throw new UndeliverableMessageException(e);
-    } catch (IOException e) {
-      warn(TAG, "Failure", e);
-      throw new RetryLaterException(e);
     }
   }
 

app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java

@@ -70,6 +70,7 @@ public final class FeatureFlags {
   private static final String AUTOMATIC_SESSION_RESET           = "android.automaticSessionReset.2";
   private static final String AUTOMATIC_SESSION_INTERVAL        = "android.automaticSessionResetInterval";
   private static final String DEFAULT_MAX_BACKOFF               = "android.defaultMaxBackoff";
+  private static final String SERVER_ERROR_MAX_BACKOFF          = "android.serverErrorMaxBackoff";
   private static final String OKHTTP_AUTOMATIC_RETRY            = "android.okhttpAutomaticRetry";
   private static final String SHARE_SELECTION_LIMIT             = "android.share.limit";
   private static final String ANIMATED_STICKER_MIN_MEMORY       = "android.animatedStickerMinMemory";
@@ -101,6 +102,7 @@ public final class FeatureFlags {
       AUTOMATIC_SESSION_RESET,
       AUTOMATIC_SESSION_INTERVAL,
       DEFAULT_MAX_BACKOFF,
+      SERVER_ERROR_MAX_BACKOFF,
       OKHTTP_AUTOMATIC_RETRY,
       SHARE_SELECTION_LIMIT,
       ANIMATED_STICKER_MIN_MEMORY,
@@ -142,6 +144,7 @@ public final class FeatureFlags {
       AUTOMATIC_SESSION_RESET,
       AUTOMATIC_SESSION_INTERVAL,
       DEFAULT_MAX_BACKOFF,
+      SERVER_ERROR_MAX_BACKOFF,
       OKHTTP_AUTOMATIC_RETRY,
       SHARE_SELECTION_LIMIT,
       ANIMATED_STICKER_MIN_MEMORY,
@@ -321,10 +324,16 @@ public final class FeatureFlags {
     return getInteger(AUTOMATIC_SESSION_RESET, (int) TimeUnit.HOURS.toSeconds(1));
   }
 
+  /** The default maximum backoff for jobs. */
   public static long getDefaultMaxBackoff() {
     return TimeUnit.SECONDS.toMillis(getInteger(DEFAULT_MAX_BACKOFF, 60));
   }
 
+  /** The maximum backoff for network jobs that hit a 5xx error. */
+  public static long getServerErrorMaxBackoff() {
+    return TimeUnit.SECONDS.toMillis(getInteger(SERVER_ERROR_MAX_BACKOFF, (int) TimeUnit.HOURS.toSeconds(6)));
+  }
+
   /** Whether or not to allow automatic retries from OkHttp */
   public static boolean okHttpAutomaticRetry() {
     return getBoolean(OKHTTP_AUTOMATIC_RETRY, false);

libsignal/service/src/main/java/org/whispersystems/signalservice/api/push/exceptions/NonSuccessfulResponseCodeException.java

@@ -27,4 +27,8 @@ public class NonSuccessfulResponseCodeException extends IOException {
   public int getCode() {
     return code;
   }
+
+  public boolean is5xx() {
+    return code >= 500 && code < 600;
+  }
 }