Remove recording retry/throttle as this is now handled in a more general way

pull/1277/head
litetex 2025-02-11 20:47:14 +01:00
rodzic 460e0f77e6
commit 9f94a2912d
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 525B43E6039B3689
1 zmienionych plików z 15 dodań i 70 usunięć

Wyświetl plik

@ -15,7 +15,6 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
import javax.annotation.Nonnull;
@ -47,20 +46,6 @@ class RecordingDownloader extends Downloader {
private int index = 0;
private final String path;
// try to prevent ReCaptchaExceptions / rate limits by tracking and throttling the requests
/**
* The maximum number of requests per 20 seconds which are executed
* by the {@link RecordingDownloader}.
* 20 seconds is used as upper bound because the rate limit can be triggered within 30 seconds
* and hitting the rate limit should be prevented because it comes with a bigger delay.
* The values can be adjusted when executing the downloader and running into problems.
* <p>TODO: Allow adjusting the value by setting a param in the gradle command</p>
*/
private static final int MAX_REQUESTS_PER_20_SECONDS = 30;
private static final long[] requestTimes = new long[MAX_REQUESTS_PER_20_SECONDS];
private static int requestTimesCursor = -1;
private static final Random throttleRandom = new Random();
/**
* Creates the folder described by {@code stringPath} if it does not exist.
* Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}.
@ -84,71 +69,31 @@ class RecordingDownloader extends Downloader {
@Override
public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException {
// Delay the execution if the max number of requests per minute is reached
final long currentTime = System.currentTimeMillis();
// the cursor points to the latest request time and the next position is the oldest one
final int oldestRequestTimeCursor = (requestTimesCursor + 1) % requestTimes.length;
final long oldestRequestTime = requestTimes[oldestRequestTimeCursor];
if (oldestRequestTime + 20_000 >= currentTime) {
try {
// sleep at least until the oldest request is 20s old, but not more than 20s
final int minSleepTime = (int) (currentTime - oldestRequestTime);
Thread.sleep(minSleepTime + throttleRandom.nextInt(20_000 - minSleepTime));
} catch (InterruptedException e) {
// handle the exception gracefully because it's not critical for the test
System.err.println("Error while throttling the RecordingDownloader.");
e.printStackTrace();
}
}
requestTimesCursor = oldestRequestTimeCursor; // the oldest value needs to be overridden
requestTimes[requestTimesCursor] = System.currentTimeMillis();
// Handle ReCaptchaExceptions by retrying the request once after a while
try {
return executeRequest(request);
} catch (ReCaptchaException e) {
try {
// sleep for 35-60 seconds to circumvent the rate limit
System.out.println("Throttling the RecordingDownloader to handle a ReCaptcha."
+ " Sleeping for 35-60 seconds.");
Thread.sleep(35_000 + throttleRandom.nextInt(25_000));
} catch (InterruptedException ie) {
// handle the exception gracefully because it's not critical for the test
System.err.println("Error while throttling the RecordingDownloader.");
ie.printStackTrace();
e.printStackTrace();
}
return executeRequest(request);
}
}
@Nonnull
private Response executeRequest(@Nonnull final Request request) throws IOException,
ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response(
response.responseCode(),
response.responseMessage(),
response.responseHeaders(),
cleanedResponseBody,
response.latestUrl()
response.responseCode(),
response.responseMessage(),
response.responseHeaders(),
response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1"),
response.latestUrl()
);
final File outputFile = new File(path + File.separator + FILE_NAME_PREFIX + index
+ ".json");
final File outputFile = new File(
path + File.separator + FILE_NAME_PREFIX + index + ".json");
index++;
outputFile.createNewFile();
final OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(outputFile),
StandardCharsets.UTF_8);
new GsonBuilder()
try (final OutputStreamWriter writer = new OutputStreamWriter(
new FileOutputStream(outputFile), StandardCharsets.UTF_8)) {
new GsonBuilder()
.setPrettyPrinting()
.create()
.toJson(new TestRequestResponse(request, response), writer);
writer.flush();
writer.close();
writer.flush();
}
return response;
}