Add support for setting max instances per job queue.

fork-5.53.8
Greyson Parrelli 2020-12-07 17:30:05 -05:00 zatwierdzone przez GitHub
rodzic dc4ce234b7
commit c3d7b88cf6
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
29 zmienionych plików z 136 dodań i 73 usunięć

Wyświetl plik

@ -177,7 +177,7 @@ public class JobDatabase extends Database {
values.put(Jobs.RUN_ATTEMPT, job.getRunAttempt());
values.put(Jobs.MAX_ATTEMPTS, job.getMaxAttempts());
values.put(Jobs.MAX_BACKOFF, job.getMaxBackoff());
values.put(Jobs.MAX_INSTANCES, job.getMaxInstances());
values.put(Jobs.MAX_INSTANCES, job.getMaxInstancesForFactory());
values.put(Jobs.LIFESPAN, job.getLifespan());
values.put(Jobs.SERIALIZED_DATA, job.getSerializedData());
values.put(Jobs.SERIALIZED_INPUT_DATA, job.getSerializedInputData());
@ -254,7 +254,7 @@ public class JobDatabase extends Database {
contentValues.put(Jobs.RUN_ATTEMPT, job.getRunAttempt());
contentValues.put(Jobs.MAX_ATTEMPTS, job.getMaxAttempts());
contentValues.put(Jobs.MAX_BACKOFF, job.getMaxBackoff());
contentValues.put(Jobs.MAX_INSTANCES, job.getMaxInstances());
contentValues.put(Jobs.MAX_INSTANCES, job.getMaxInstancesForFactory());
contentValues.put(Jobs.LIFESPAN, job.getLifespan());
contentValues.put(Jobs.SERIALIZED_DATA, job.getSerializedData());
contentValues.put(Jobs.SERIALIZED_INPUT_DATA, job.getSerializedInputData());

Wyświetl plik

@ -241,7 +241,8 @@ public abstract class Job {
private final long lifespan;
private final int maxAttempts;
private final long maxBackoff;
private final int maxInstances;
private final int maxInstancesForFactory;
private final int maxInstancesForQueue;
private final String queue;
private final List<String> constraintKeys;
private final Data inputData;
@ -252,22 +253,24 @@ public abstract class Job {
long lifespan,
int maxAttempts,
long maxBackoff,
int maxInstances,
int maxInstancesForFactory,
int maxInstancesForQueue,
@Nullable String queue,
@NonNull List<String> constraintKeys,
@Nullable Data inputData,
boolean memoryOnly)
{
this.id = id;
this.createTime = createTime;
this.lifespan = lifespan;
this.maxAttempts = maxAttempts;
this.maxBackoff = maxBackoff;
this.maxInstances = maxInstances;
this.queue = queue;
this.constraintKeys = constraintKeys;
this.inputData = inputData;
this.memoryOnly = memoryOnly;
this.id = id;
this.createTime = createTime;
this.lifespan = lifespan;
this.maxAttempts = maxAttempts;
this.maxBackoff = maxBackoff;
this.maxInstancesForFactory = maxInstancesForFactory;
this.maxInstancesForQueue = maxInstancesForQueue;
this.queue = queue;
this.constraintKeys = constraintKeys;
this.inputData = inputData;
this.memoryOnly = memoryOnly;
}
@NonNull String getId() {
@ -290,8 +293,12 @@ public abstract class Job {
return maxBackoff;
}
int getMaxInstances() {
return maxInstances;
int getMaxInstancesForFactory() {
return maxInstancesForFactory;
}
int getMaxInstancesForQueue() {
return maxInstancesForQueue;
}
public @Nullable String getQueue() {
@ -311,18 +318,18 @@ public abstract class Job {
}
public Builder toBuilder() {
return new Builder(id, createTime, maxBackoff, lifespan, maxAttempts, maxInstances, queue, constraintKeys, inputData, memoryOnly);
return new Builder(id, createTime, maxBackoff, lifespan, maxAttempts, maxInstancesForFactory, maxInstancesForQueue, queue, constraintKeys, inputData, memoryOnly);
}
public static final class Builder {
private String id;
private long createTime;
private long maxBackoff;
private long lifespan;
private int maxAttempts;
private int maxInstances;
private int maxInstancesForFactory;
private int maxInstancesForQueue;
private String queue;
private List<String> constraintKeys;
private Data inputData;
@ -333,7 +340,7 @@ public abstract class Job {
}
Builder(@NonNull String id) {
this(id, System.currentTimeMillis(), TimeUnit.SECONDS.toMillis(30), IMMORTAL, 1, UNLIMITED, null, new LinkedList<>(), null, false);
this(id, System.currentTimeMillis(), TimeUnit.SECONDS.toMillis(30), IMMORTAL, 1, UNLIMITED, UNLIMITED, null, new LinkedList<>(), null, false);
}
private Builder(@NonNull String id,
@ -341,22 +348,24 @@ public abstract class Job {
long maxBackoff,
long lifespan,
int maxAttempts,
int maxInstances,
int maxInstancesForFactory,
int maxInstancesForQueue,
@Nullable String queue,
@NonNull List<String> constraintKeys,
@Nullable Data inputData,
boolean memoryOnly)
{
this.id = id;
this.createTime = createTime;
this.maxBackoff = maxBackoff;
this.lifespan = lifespan;
this.maxAttempts = maxAttempts;
this.maxInstances = maxInstances;
this.queue = queue;
this.constraintKeys = constraintKeys;
this.inputData = inputData;
this.memoryOnly = memoryOnly;
this.id = id;
this.createTime = createTime;
this.maxBackoff = maxBackoff;
this.lifespan = lifespan;
this.maxAttempts = maxAttempts;
this.maxInstancesForFactory = maxInstancesForFactory;
this.maxInstancesForQueue = maxInstancesForQueue;
this.queue = queue;
this.constraintKeys = constraintKeys;
this.inputData = inputData;
this.memoryOnly = memoryOnly;
}
/** Should only be invoked by {@link JobController} */
@ -391,17 +400,31 @@ public abstract class Job {
}
/**
* Specify the maximum number of instances you'd want of this job at any given time. If
* enqueueing this job would put it over that limit, it will be ignored.
*
* Duplicates are determined by two jobs having the same {@link Job#getFactoryKey()}.
* Specify the maximum number of instances you'd want of this job at any given time, as
* determined by the job's factory key. If enqueueing this job would put it over that limit,
* it will be ignored.
*
* This property is ignored if the job is submitted as part of a {@link JobManager.Chain}.
*
* Defaults to {@link #UNLIMITED}.
*/
public @NonNull Builder setMaxInstances(int maxInstances) {
this.maxInstances = maxInstances;
public @NonNull Builder setMaxInstancesForFactory(int maxInstancesForFactory) {
this.maxInstancesForFactory = maxInstancesForFactory;
return this;
}
/**
* Specify the maximum number of instances you'd want of this job at any given time, as
* determined by the job's queue key. If enqueueing this job would put it over that limit,
* it will be ignored.
*
* This property is ignored if the job is submitted as part of a {@link JobManager.Chain}, or
* if the job has no queue key.
*
* Defaults to {@link #UNLIMITED}.
*/
public @NonNull Builder setMaxInstancesForQueue(int maxInstancesForQueue) {
this.maxInstancesForQueue = maxInstancesForQueue;
return this;
}
@ -455,7 +478,7 @@ public abstract class Job {
}
public @NonNull Parameters build() {
return new Parameters(id, createTime, lifespan, maxAttempts, maxBackoff, maxInstances, queue, constraintKeys, inputData, memoryOnly);
return new Parameters(id, createTime, lifespan, maxAttempts, maxBackoff, maxInstancesForFactory, maxInstancesForQueue, queue, constraintKeys, inputData, memoryOnly);
}
}
}

Wyświetl plik

@ -89,7 +89,7 @@ class JobController {
if (chainExceedsMaximumInstances(chain)) {
Job solo = chain.get(0).get(0);
jobTracker.onStateChange(solo, JobTracker.JobState.IGNORED);
Log.w(TAG, JobLogger.format(solo, "Already at the max instance count of " + solo.getParameters().getMaxInstances() + ". Skipping."));
Log.w(TAG, JobLogger.format(solo, "Already at the max instance count. Factory limit: " + solo.getParameters().getMaxInstancesForFactory() + ", Queue limit: " + solo.getParameters().getMaxInstancesForQueue() + ". Skipping."));
return;
}
@ -105,7 +105,7 @@ class JobController {
if (chainExceedsMaximumInstances(chain)) {
jobTracker.onStateChange(job, JobTracker.JobState.IGNORED);
Log.w(TAG, JobLogger.format(job, "Already at the max instance count of " + job.getParameters().getMaxInstances() + ". Skipping."));
Log.w(TAG, JobLogger.format(job, "Already at the max instance count. Factory limit: " + job.getParameters().getMaxInstancesForFactory() + ", Queue limit: " + job.getParameters().getMaxInstancesForQueue() + ". Skipping."));
return;
}
@ -299,12 +299,22 @@ class JobController {
if (chain.size() == 1 && chain.get(0).size() == 1) {
Job solo = chain.get(0).get(0);
if (solo.getParameters().getMaxInstances() != Job.Parameters.UNLIMITED &&
jobStorage.getJobInstanceCount(solo.getFactoryKey()) >= solo.getParameters().getMaxInstances())
{
boolean exceedsFactory = solo.getParameters().getMaxInstancesForFactory() != Job.Parameters.UNLIMITED &&
jobStorage.getJobCountForFactory(solo.getFactoryKey()) >= solo.getParameters().getMaxInstancesForFactory();
if (exceedsFactory) {
return true;
}
boolean exceedsQueue = solo.getParameters().getQueue() != null &&
solo.getParameters().getMaxInstancesForQueue() != Job.Parameters.UNLIMITED &&
jobStorage.getJobCountForQueue(solo.getParameters().getQueue()) >= solo.getParameters().getMaxInstancesForQueue();
if (exceedsQueue) {
return true;
}
}
return false;
}
@ -345,7 +355,7 @@ class JobController {
job.getParameters().getMaxAttempts(),
job.getParameters().getMaxBackoff(),
job.getParameters().getLifespan(),
job.getParameters().getMaxInstances(),
job.getParameters().getMaxInstancesForFactory(),
dataSerializer.serialize(job.serialize()),
null,
false,
@ -459,7 +469,7 @@ class JobController {
jobSpec.getMaxAttempts(),
jobSpec.getMaxBackoff(),
jobSpec.getLifespan(),
jobSpec.getMaxInstances(),
jobSpec.getMaxInstancesForFactory(),
jobSpec.getSerializedData(),
dataSerializer.serialize(inputData),
jobSpec.isRunning(),

Wyświetl plik

@ -71,7 +71,7 @@ public class JobMigrator {
jobSpec.getMaxAttempts(),
jobSpec.getMaxBackoff(),
jobSpec.getLifespan(),
jobSpec.getMaxInstances(),
jobSpec.getMaxInstancesForFactory(),
dataSerializer.serialize(updatedJobData.getData()),
jobSpec.getSerializedInputData(),
jobSpec.isRunning(),

Wyświetl plik

@ -87,7 +87,7 @@ public final class JobSpec {
return maxBackoff;
}
public int getMaxInstances() {
public int getMaxInstancesForFactory() {
return maxInstances;
}

Wyświetl plik

@ -27,7 +27,10 @@ public interface JobStorage {
@NonNull List<JobSpec> getJobsInQueue(@NonNull String queue);
@WorkerThread
int getJobInstanceCount(@NonNull String factoryKey);
int getJobCountForFactory(@NonNull String factoryKey);
@WorkerThread
int getJobCountForQueue(@NonNull String queueKey);
@WorkerThread
void updateJobRunningState(@NonNull String id, boolean isRunning);

Wyświetl plik

@ -29,7 +29,7 @@ public class ConversationShortcutUpdateJob extends BaseJob {
this(new Parameters.Builder()
.setQueue("ConversationShortcutUpdateJob")
.setLifespan(TimeUnit.MINUTES.toMillis(15))
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.build());
}

Wyświetl plik

@ -144,12 +144,19 @@ public class FastJobStorage implements JobStorage {
}
@Override
public synchronized int getJobInstanceCount(@NonNull String factoryKey) {
public synchronized int getJobCountForFactory(@NonNull String factoryKey) {
return (int) Stream.of(jobs)
.filter(j -> j.getFactoryKey().equals(factoryKey))
.count();
}
@Override
public synchronized int getJobCountForQueue(@NonNull String queueKey) {
return (int) Stream.of(jobs)
.filter(j -> j.getQueueKey().equals(queueKey))
.count();
}
@Override
public synchronized void updateJobRunningState(@NonNull String id, boolean isRunning) {
JobSpec job = getJobById(id);
@ -171,7 +178,7 @@ public class FastJobStorage implements JobStorage {
existing.getMaxAttempts(),
existing.getMaxBackoff(),
existing.getLifespan(),
existing.getMaxInstances(),
existing.getMaxInstancesForFactory(),
existing.getSerializedData(),
existing.getSerializedInputData(),
isRunning,
@ -202,7 +209,7 @@ public class FastJobStorage implements JobStorage {
existing.getMaxAttempts(),
existing.getMaxBackoff(),
existing.getLifespan(),
existing.getMaxInstances(),
existing.getMaxInstancesForFactory(),
serializedData,
existing.getSerializedInputData(),
isRunning,
@ -229,7 +236,7 @@ public class FastJobStorage implements JobStorage {
existing.getMaxAttempts(),
existing.getMaxBackoff(),
existing.getLifespan(),
existing.getMaxInstances(),
existing.getMaxInstancesForFactory(),
existing.getSerializedData(),
existing.getSerializedInputData(),
false,

Wyświetl plik

@ -58,7 +58,7 @@ public class FcmRefreshJob extends BaseJob {
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(1)
.setLifespan(TimeUnit.MINUTES.toMillis(5))
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.build());
}

Wyświetl plik

@ -29,7 +29,7 @@ public class KbsEnclaveMigrationWorkerJob extends BaseJob {
.setLifespan(Parameters.IMMORTAL)
.setMaxAttempts(Parameters.UNLIMITED)
.setQueue("KbsEnclaveMigrationWorkerJob")
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.build());
}

Wyświetl plik

@ -46,7 +46,7 @@ public final class LocalBackupJob extends BaseJob {
JobManager jobManager = ApplicationDependencies.getJobManager();
Parameters.Builder parameters = new Parameters.Builder()
.setQueue(QUEUE)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setMaxAttempts(3);
if (force) {
jobManager.cancelAllInQueue(QUEUE);

Wyświetl plik

@ -30,7 +30,7 @@ public class MultiDeviceKeysUpdateJob extends BaseJob {
public MultiDeviceKeysUpdateJob() {
this(new Parameters.Builder()
.setQueue("MultiDeviceKeysUpdateJob")
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(10)
.build());

Wyświetl plik

@ -22,7 +22,7 @@ public class MultiDeviceProfileContentUpdateJob extends BaseJob {
public MultiDeviceProfileContentUpdateJob() {
this(new Parameters.Builder()
.setQueue("MultiDeviceProfileUpdateJob")
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(10)
.build());

Wyświetl plik

@ -22,7 +22,7 @@ public class MultiDeviceStorageSyncRequestJob extends BaseJob {
public MultiDeviceStorageSyncRequestJob() {
this(new Parameters.Builder()
.setQueue("MultiDeviceStorageSyncRequestJob")
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(10)
.build());

Wyświetl plik

@ -38,7 +38,7 @@ public final class ProfileUploadJob extends BaseJob {
.setQueue(QUEUE)
.setLifespan(TimeUnit.DAYS.toMillis(30))
.setMaxAttempts(Parameters.UNLIMITED)
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.build());
}

Wyświetl plik

@ -26,7 +26,7 @@ public class PushNotificationReceiveJob extends BaseJob {
.addConstraint(NetworkConstraint.KEY)
.setQueue("__notification_received")
.setMaxAttempts(3)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.build());
setContext(context);
}

Wyświetl plik

@ -31,7 +31,7 @@ public class RefreshAttributesJob extends BaseJob {
this(new Job.Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setQueue("RefreshAttributesJob")
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.build());
}

Wyświetl plik

@ -43,7 +43,7 @@ public class RefreshOwnProfileJob extends BaseJob {
this(new Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setQueue(ProfileUploadJob.QUEUE)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setMaxAttempts(10)
.build());
}

Wyświetl plik

@ -36,7 +36,7 @@ public class RefreshPreKeysJob extends BaseJob {
this(new Job.Parameters.Builder()
.setQueue("RefreshPreKeysJob")
.addConstraint(NetworkConstraint.KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setMaxAttempts(Parameters.UNLIMITED)
.setLifespan(TimeUnit.DAYS.toMillis(30))
.build());

Wyświetl plik

@ -23,7 +23,7 @@ public class RemoteConfigRefreshJob extends BaseJob {
public RemoteConfigRefreshJob() {
this(new Job.Parameters.Builder()
.setQueue("RemoteConfigRefreshJob")
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(Parameters.UNLIMITED)
.setLifespan(TimeUnit.DAYS.toMillis(1))

Wyświetl plik

@ -20,7 +20,7 @@ public class RotateProfileKeyJob extends BaseJob {
public RotateProfileKeyJob() {
this(new Job.Parameters.Builder()
.setQueue("__ROTATE_PROFILE_KEY__")
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.build());
}

Wyświetl plik

@ -28,7 +28,7 @@ public class RotateSignedPreKeyJob extends BaseJob {
this(new Job.Parameters.Builder()
.setQueue("RotateSignedPreKeyJob")
.addConstraint(NetworkConstraint.KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setMaxAttempts(Parameters.UNLIMITED)
.setLifespan(TimeUnit.DAYS.toMillis(2))
.build());

Wyświetl plik

@ -30,7 +30,7 @@ public class ServiceOutageDetectionJob extends BaseJob {
.setQueue("ServiceOutageDetectionJob")
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(5)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.build());
}

Wyświetl plik

@ -41,7 +41,7 @@ public class StorageAccountRestoreJob extends BaseJob {
this(new Parameters.Builder()
.setQueue(StorageSyncJob.QUEUE_KEY)
.addConstraint(NetworkConstraint.KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setMaxAttempts(1)
.setLifespan(LIFESPAN)
.build());

Wyświetl plik

@ -51,7 +51,7 @@ public class StorageForcePushJob extends BaseJob {
public StorageForcePushJob() {
this(new Parameters.Builder().addConstraint(NetworkConstraint.KEY)
.setQueue(StorageSyncJob.QUEUE_KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.build());
}

Wyświetl plik

@ -75,7 +75,7 @@ public class StorageSyncJob extends BaseJob {
public StorageSyncJob() {
this(new Job.Parameters.Builder().addConstraint(NetworkConstraint.KEY)
.setQueue(QUEUE_KEY)
.setMaxInstances(2)
.setMaxInstancesForFactory(2)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.build());
}

Wyświetl plik

@ -22,7 +22,7 @@ abstract class MigrationJob extends Job {
MigrationJob(@NonNull Parameters parameters) {
super(parameters.toBuilder()
.setQueue(Parameters.MIGRATION_QUEUE_KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.setLifespan(Parameters.IMMORTAL)
.setMaxAttempts(Parameters.UNLIMITED)
.build());

Wyświetl plik

@ -33,7 +33,7 @@ public final class RegistrationPinV2MigrationJob extends BaseJob {
public RegistrationPinV2MigrationJob() {
this(new Parameters.Builder()
.setQueue(KEY)
.setMaxInstances(1)
.setMaxInstancesForFactory(1)
.addConstraint(NetworkConstraint.KEY)
.setLifespan(Job.Parameters.IMMORTAL)
.setMaxAttempts(Job.Parameters.UNLIMITED)

Wyświetl plik

@ -536,6 +536,26 @@ public class FastJobStorageTest {
assertEquals("id1", result.get(0).getId());
}
@Test
public void getJobCountForFactory_general() {
FastJobStorage subject = new FastJobStorage(fixedDataDatabase(DataSet1.FULL_SPECS));
subject.init();
assertEquals(1, subject.getJobCountForFactory("f1"));
assertEquals(0, subject.getJobCountForFactory("does-not-exist"));
}
@Test
public void getJobCountForQueue_general() {
FastJobStorage subject = new FastJobStorage(fixedDataDatabase(DataSet1.FULL_SPECS));
subject.init();
assertEquals(1, subject.getJobCountForQueue("q1"));
assertEquals(0, subject.getJobCountForQueue("does-not-exist"));
}
private JobDatabase noopDatabase() {
JobDatabase database = mock(JobDatabase.class);