2021-12-23 10:42:24 +00:00
|
|
|
package com.onthegomap.planetiler.util;
|
2021-09-10 00:46:20 +00:00
|
|
|
|
2021-11-29 12:41:57 +00:00
|
|
|
import static com.google.common.net.HttpHeaders.*;
|
2021-10-20 01:57:47 +00:00
|
|
|
import static java.nio.file.StandardOpenOption.WRITE;
|
2021-09-10 00:46:20 +00:00
|
|
|
|
2023-01-30 18:38:09 +00:00
|
|
|
import com.google.common.util.concurrent.RateLimiter;
|
2021-12-23 10:42:24 +00:00
|
|
|
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
|
|
|
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
|
|
|
import com.onthegomap.planetiler.stats.Stats;
|
|
|
|
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.io.IOException;
|
2021-10-20 01:57:47 +00:00
|
|
|
import java.io.InputStream;
|
|
|
|
import java.io.UncheckedIOException;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.net.URI;
|
2021-10-20 01:57:47 +00:00
|
|
|
import java.net.URLConnection;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.net.http.HttpClient;
|
2021-10-20 01:57:47 +00:00
|
|
|
import java.net.http.HttpHeaders;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.net.http.HttpRequest;
|
|
|
|
import java.net.http.HttpResponse;
|
2021-10-20 01:57:47 +00:00
|
|
|
import java.nio.ByteBuffer;
|
|
|
|
import java.nio.channels.Channels;
|
|
|
|
import java.nio.channels.FileChannel;
|
|
|
|
import java.nio.channels.ReadableByteChannel;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.nio.file.Files;
|
|
|
|
import java.nio.file.Path;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
2021-11-29 12:41:57 +00:00
|
|
|
import java.util.Optional;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.util.concurrent.CompletableFuture;
|
|
|
|
import java.util.concurrent.ExecutionException;
|
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
|
import java.util.concurrent.Executors;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import java.util.concurrent.TimeoutException;
|
2021-10-20 01:57:47 +00:00
|
|
|
import java.util.concurrent.atomic.AtomicLong;
|
2021-09-10 00:46:20 +00:00
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A utility for downloading files to disk in parallel over HTTP.
|
|
|
|
* <p>
|
|
|
|
* After downloading a file once, it won't be downloaded again unless the {@code Content-Length} of the resource
|
|
|
|
* changes.
|
|
|
|
* <p>
|
|
|
|
* For example:
|
2022-03-19 09:46:03 +00:00
|
|
|
*
|
2022-03-09 02:08:03 +00:00
|
|
|
* <pre>
|
|
|
|
* {@code
|
2021-12-23 10:42:24 +00:00
|
|
|
* Downloader.create(PlanetilerConfig.defaults())
|
2021-09-10 00:46:20 +00:00
|
|
|
* .add("natural_earth", "http://url/of/natural_earth.zip", Path.of("natural_earth.zip"))
|
|
|
|
* .add("osm", "http://url/of/file.osm.pbf", Path.of("file.osm.pbf"))
|
2022-03-03 12:25:24 +00:00
|
|
|
* .run();
|
2022-03-09 02:08:03 +00:00
|
|
|
* }
|
|
|
|
* </pre>
|
2021-09-10 00:46:20 +00:00
|
|
|
* <p>
|
|
|
|
* As a shortcut to find the URL of a file to download from the <a href="https://download.geofabrik.de/">Geofabrik
|
|
|
|
* download site</a>, you can use "geofabrik:extract name" (i.e. "geofabrik:monaco" or "geofabrik:australia") to look up
|
|
|
|
* a {@code .osm.pbf} download URL in the <a href="https://download.geofabrik.de/technical.html">Geofabrik JSON
|
|
|
|
* index</a>.
|
2021-10-20 01:57:47 +00:00
|
|
|
* <p>
|
2023-06-08 16:01:01 +00:00
|
|
|
* Use "aws:latest" to download the latest {@code planet.osm.pbf} file from the
|
|
|
|
* <a href="https://registry.opendata.aws/osm/">AWS Open Data Registry</a>, or "overture:latest" to download the latest
|
|
|
|
* <a href="https://overturemaps.org/">Overture Maps Foundation</a> release.
|
2021-09-10 00:46:20 +00:00
|
|
|
*/
|
|
|
|
@SuppressWarnings("UnusedReturnValue")
|
|
|
|
public class Downloader {
|
|
|
|
|
2021-11-29 12:41:57 +00:00
|
|
|
private static final int MAX_REDIRECTS = 5;
|
2021-09-10 00:46:20 +00:00
|
|
|
private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
|
2021-12-23 10:42:24 +00:00
|
|
|
private final PlanetilerConfig config;
|
2021-09-10 00:46:20 +00:00
|
|
|
private final List<ResourceToDownload> toDownloadList = new ArrayList<>();
|
2021-11-29 12:41:57 +00:00
|
|
|
private final HttpClient client = HttpClient.newBuilder()
|
|
|
|
// explicitly follow redirects to capture final redirect url
|
|
|
|
.followRedirects(HttpClient.Redirect.NEVER).build();
|
2021-09-10 00:46:20 +00:00
|
|
|
private final ExecutorService executor;
|
2021-10-20 01:57:47 +00:00
|
|
|
private final Stats stats;
|
|
|
|
private final long chunkSizeBytes;
|
2022-03-19 09:46:03 +00:00
|
|
|
private final ResourceUsage diskSpaceCheck = new ResourceUsage("download");
|
2023-01-30 18:38:09 +00:00
|
|
|
private final RateLimiter rateLimiter;
|
2021-09-10 00:46:20 +00:00
|
|
|
|
2021-12-23 10:42:24 +00:00
|
|
|
Downloader(PlanetilerConfig config, Stats stats, long chunkSizeBytes) {
|
2023-01-30 18:38:09 +00:00
|
|
|
this.rateLimiter = config.downloadMaxBandwidth() == 0 ? null : RateLimiter.create(config.downloadMaxBandwidth());
|
2021-10-20 01:57:47 +00:00
|
|
|
this.chunkSizeBytes = chunkSizeBytes;
|
2021-09-10 00:46:20 +00:00
|
|
|
this.config = config;
|
2021-10-20 01:57:47 +00:00
|
|
|
this.stats = stats;
|
2022-04-23 09:58:49 +00:00
|
|
|
this.executor = Executors.newSingleThreadExecutor(runnable -> {
|
2021-09-10 00:46:20 +00:00
|
|
|
Thread thread = new Thread(() -> {
|
|
|
|
LogUtil.setStage("download");
|
|
|
|
runnable.run();
|
|
|
|
});
|
|
|
|
thread.setDaemon(true);
|
|
|
|
return thread;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2021-12-23 10:42:24 +00:00
|
|
|
public static Downloader create(PlanetilerConfig config, Stats stats) {
|
2021-10-20 01:57:47 +00:00
|
|
|
return new Downloader(config, stats, config.downloadChunkSizeMB() * 1_000_000L);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
|
2021-12-23 10:42:24 +00:00
|
|
|
private static URLConnection getUrlConnection(String urlString, PlanetilerConfig config) throws IOException {
|
2023-09-22 01:44:09 +00:00
|
|
|
var url = URI.create(urlString).toURL();
|
2021-10-20 01:57:47 +00:00
|
|
|
var connection = url.openConnection();
|
|
|
|
connection.setConnectTimeout((int) config.httpTimeout().toMillis());
|
|
|
|
connection.setReadTimeout((int) config.httpTimeout().toMillis());
|
|
|
|
connection.setRequestProperty(USER_AGENT, config.httpUserAgent());
|
|
|
|
return connection;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2021-12-23 10:42:24 +00:00
|
|
|
* Returns an input stream reading from a remote URL with timeout and user-agent set from planetiler config.
|
2021-10-20 01:57:47 +00:00
|
|
|
*
|
|
|
|
* @param urlString remote URL
|
2021-12-23 10:42:24 +00:00
|
|
|
* @param config planetiler config containing the user agent and timeout parameter
|
2021-10-20 01:57:47 +00:00
|
|
|
* @return an input stream that will read from the remote URL
|
|
|
|
* @throws IOException if an error occurs making the network request
|
|
|
|
*/
|
2021-12-23 10:42:24 +00:00
|
|
|
public static InputStream openStream(String urlString, PlanetilerConfig config) throws IOException {
|
2021-10-20 01:57:47 +00:00
|
|
|
return getUrlConnection(urlString, config).getInputStream();
|
|
|
|
}
|
|
|
|
|
2021-12-23 10:42:24 +00:00
|
|
|
private static InputStream openStreamRange(String urlString, PlanetilerConfig config, long start, long end)
|
2021-10-20 01:57:47 +00:00
|
|
|
throws IOException {
|
|
|
|
URLConnection connection = getUrlConnection(urlString, config);
|
|
|
|
connection.setRequestProperty(RANGE, "bytes=%d-%d".formatted(start, end));
|
|
|
|
return connection.getInputStream();
|
|
|
|
}
|
|
|
|
|
|
|
|
InputStream openStream(String url) throws IOException {
|
|
|
|
return openStream(url, config);
|
|
|
|
}
|
|
|
|
|
|
|
|
InputStream openStreamRange(String url, long start, long end) throws IOException {
|
|
|
|
return openStreamRange(url, config, start, end);
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* Adds a new resource to download but does not start downloading it until {@link #run()} is called.
|
|
|
|
* <p>
|
|
|
|
* The resource won't be downloaded if size on disk is the same as {@code Content-Length} header reported from a
|
|
|
|
* {@code HEAD} request to the resource.
|
|
|
|
*
|
|
|
|
* @param id short name to use for this download when logging progress
|
2023-06-08 16:01:01 +00:00
|
|
|
* @param url the external resource to fetch, "aws:latest" (for the latest planet .osm.pbf), "overture:latest" (for
|
|
|
|
* the latest Overture Maps release) or "geofabrik:extract-name" as a shortcut to use
|
|
|
|
* {@link Geofabrik#getDownloadUrl(String, PlanetilerConfig)} to look up a {@code .osm.pbf}
|
|
|
|
* <a href="https://download.geofabrik.de/">Geofabrik</a> extract URL by partial match on area name
|
2021-09-10 00:46:20 +00:00
|
|
|
* @param output where to download the file to
|
|
|
|
* @return {@code this} for chaining
|
|
|
|
*/
|
|
|
|
public Downloader add(String id, String url, Path output) {
|
|
|
|
if (url.startsWith("geofabrik:")) {
|
2021-10-20 01:57:47 +00:00
|
|
|
url = Geofabrik.getDownloadUrl(url.replaceFirst("^geofabrik:", ""), config);
|
|
|
|
} else if (url.startsWith("aws:")) {
|
2023-06-08 16:01:01 +00:00
|
|
|
url = AwsOsm.OSM_PDS.getDownloadUrl(url.replaceFirst("^aws:", ""), config);
|
|
|
|
} else if (url.startsWith("overture:")) {
|
|
|
|
url = AwsOsm.OVERTURE.getDownloadUrl(url.replaceFirst("^overture:", ""), config);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
toDownloadList.add(new ResourceToDownload(id, url, output));
|
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Starts downloading all resources in parallel, logging progress until complete.
|
|
|
|
*
|
|
|
|
* @throws IllegalStateException if an error occurs downloading any resource, will be thrown after all resources
|
|
|
|
* finish
|
|
|
|
*/
|
|
|
|
public void run() {
|
|
|
|
var downloads = CompletableFuture
|
|
|
|
.allOf(toDownloadList.stream()
|
|
|
|
.map(this::downloadIfNecessary)
|
|
|
|
.toArray(CompletableFuture[]::new)
|
|
|
|
);
|
|
|
|
|
|
|
|
ProgressLoggers loggers = ProgressLoggers.create();
|
|
|
|
|
|
|
|
for (var toDownload : toDownloadList) {
|
|
|
|
try {
|
2021-10-20 01:57:47 +00:00
|
|
|
long size = toDownload.metadata.get(10, TimeUnit.SECONDS).size;
|
2022-03-01 01:52:30 +00:00
|
|
|
loggers.addStorageRatePercentCounter(toDownload.id, size, toDownload::bytesDownloaded, true);
|
2022-04-23 09:58:49 +00:00
|
|
|
} catch (InterruptedException e) {
|
|
|
|
Thread.currentThread().interrupt();
|
|
|
|
throw new IllegalStateException("Error getting size of " + toDownload.url, e);
|
|
|
|
} catch (ExecutionException | TimeoutException e) {
|
2021-09-10 00:46:20 +00:00
|
|
|
throw new IllegalStateException("Error getting size of " + toDownload.url, e);
|
|
|
|
}
|
|
|
|
}
|
2022-03-04 01:21:25 +00:00
|
|
|
loggers.add(" ").addProcessStats()
|
|
|
|
.awaitAndLog(downloads, config.logInterval());
|
2021-09-10 00:46:20 +00:00
|
|
|
executor.shutdown();
|
|
|
|
}
|
|
|
|
|
2022-04-23 09:58:49 +00:00
|
|
|
CompletableFuture<Void> downloadIfNecessary(ResourceToDownload resourceToDownload) {
|
2021-09-10 00:46:20 +00:00
|
|
|
long existingSize = FileUtils.size(resourceToDownload.output);
|
|
|
|
|
2021-11-29 12:41:57 +00:00
|
|
|
return httpHeadFollowRedirects(resourceToDownload.url, 0)
|
2021-10-20 01:57:47 +00:00
|
|
|
.whenComplete((metadata, err) -> {
|
|
|
|
if (metadata != null) {
|
|
|
|
resourceToDownload.metadata.complete(metadata);
|
2021-09-10 00:46:20 +00:00
|
|
|
} else {
|
2021-10-20 01:57:47 +00:00
|
|
|
resourceToDownload.metadata.completeExceptionally(err);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
})
|
2021-10-20 01:57:47 +00:00
|
|
|
.thenComposeAsync(metadata -> {
|
|
|
|
if (metadata.size == existingSize) {
|
2022-04-23 09:58:49 +00:00
|
|
|
LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output);
|
2021-09-10 00:46:20 +00:00
|
|
|
return CompletableFuture.completedFuture(null);
|
|
|
|
} else {
|
2022-03-09 02:08:03 +00:00
|
|
|
String redirectInfo = metadata.canonicalUrl.equals(resourceToDownload.url) ? "" :
|
|
|
|
" (redirected to " + metadata.canonicalUrl + ")";
|
2022-04-23 09:58:49 +00:00
|
|
|
LOGGER.info("Downloading {}{} to {}", resourceToDownload.url, redirectInfo, resourceToDownload.output);
|
2021-09-10 00:46:20 +00:00
|
|
|
FileUtils.delete(resourceToDownload.output);
|
|
|
|
FileUtils.createParentDirectories(resourceToDownload.output);
|
|
|
|
Path tmpPath = resourceToDownload.tmpPath();
|
|
|
|
FileUtils.delete(tmpPath);
|
|
|
|
FileUtils.deleteOnExit(tmpPath);
|
2022-03-19 09:46:03 +00:00
|
|
|
diskSpaceCheck.addDisk(tmpPath, metadata.size, resourceToDownload.id);
|
|
|
|
diskSpaceCheck.checkAgainstLimits(config.force(), false);
|
2021-10-20 01:57:47 +00:00
|
|
|
return httpDownload(resourceToDownload, tmpPath)
|
2021-09-10 00:46:20 +00:00
|
|
|
.thenCompose(result -> {
|
|
|
|
try {
|
|
|
|
Files.move(tmpPath, resourceToDownload.output);
|
2022-04-23 09:58:49 +00:00
|
|
|
return CompletableFuture.completedFuture(null);
|
2021-09-10 00:46:20 +00:00
|
|
|
} catch (IOException e) {
|
2022-04-23 09:58:49 +00:00
|
|
|
return CompletableFuture.<Void>failedFuture(e);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.whenCompleteAsync((result, error) -> {
|
2021-10-20 01:57:47 +00:00
|
|
|
if (error != null) {
|
2022-04-23 09:58:49 +00:00
|
|
|
LOGGER.error("Error downloading {} to {}", resourceToDownload.url, resourceToDownload.output, error);
|
2021-10-20 01:57:47 +00:00
|
|
|
} else {
|
2022-04-23 09:58:49 +00:00
|
|
|
LOGGER.info("Finished downloading {} to {}", resourceToDownload.url, resourceToDownload.output);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
FileUtils.delete(tmpPath);
|
|
|
|
}, executor);
|
|
|
|
}
|
|
|
|
}, executor);
|
|
|
|
}
|
|
|
|
|
2021-11-29 12:41:57 +00:00
|
|
|
private CompletableFuture<ResourceMetadata> httpHeadFollowRedirects(String url, int redirects) {
|
|
|
|
if (redirects > MAX_REDIRECTS) {
|
|
|
|
throw new IllegalStateException("Exceeded " + redirects + " redirects for " + url);
|
|
|
|
}
|
2022-03-09 02:08:03 +00:00
|
|
|
return httpHead(url).thenComposeAsync(response -> response.redirect.isPresent() ?
|
|
|
|
httpHeadFollowRedirects(response.redirect.get(), redirects + 1) : CompletableFuture.completedFuture(response));
|
2021-11-29 12:41:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
CompletableFuture<ResourceMetadata> httpHead(String url) {
|
2021-09-10 00:46:20 +00:00
|
|
|
return client
|
2021-11-29 12:41:57 +00:00
|
|
|
.sendAsync(newHttpRequest(url).method("HEAD", HttpRequest.BodyPublishers.noBody()).build(),
|
2021-09-10 00:46:20 +00:00
|
|
|
responseInfo -> {
|
2021-11-29 12:41:57 +00:00
|
|
|
int status = responseInfo.statusCode();
|
|
|
|
Optional<String> location = Optional.empty();
|
|
|
|
long contentLength = 0;
|
2021-10-20 01:57:47 +00:00
|
|
|
HttpHeaders headers = responseInfo.headers();
|
2021-11-29 12:41:57 +00:00
|
|
|
if (status >= 300 && status < 400) {
|
|
|
|
location = responseInfo.headers().firstValue(LOCATION);
|
|
|
|
if (location.isEmpty()) {
|
|
|
|
throw new IllegalStateException("Received " + status + " but no location header from " + url);
|
|
|
|
}
|
|
|
|
} else if (responseInfo.statusCode() != 200) {
|
|
|
|
throw new IllegalStateException("Bad response: " + responseInfo.statusCode());
|
|
|
|
} else {
|
|
|
|
contentLength = headers.firstValueAsLong(CONTENT_LENGTH).orElseThrow();
|
|
|
|
}
|
2021-10-20 01:57:47 +00:00
|
|
|
boolean supportsRangeRequest = headers.allValues(ACCEPT_RANGES).contains("bytes");
|
2021-11-29 12:41:57 +00:00
|
|
|
ResourceMetadata metadata = new ResourceMetadata(location, url, contentLength, supportsRangeRequest);
|
2021-10-20 01:57:47 +00:00
|
|
|
return HttpResponse.BodyHandlers.replacing(metadata).apply(responseInfo);
|
2022-03-09 02:08:03 +00:00
|
|
|
})
|
|
|
|
.thenApply(HttpResponse::body);
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
|
2021-10-20 01:57:47 +00:00
|
|
|
private CompletableFuture<?> httpDownload(ResourceToDownload resource, Path tmpPath) {
|
|
|
|
/*
|
|
|
|
* Alternative using async HTTP client:
|
|
|
|
*
|
|
|
|
* return client.sendAsync(newHttpRequest(url).GET().build(), responseInfo -> {
|
|
|
|
* assertOK(responseInfo);
|
|
|
|
* return HttpResponse.BodyHandlers.ofFile(path).apply(responseInfo);
|
|
|
|
*
|
|
|
|
* But it is slower on large files
|
|
|
|
*/
|
|
|
|
return resource.metadata.thenCompose(metadata -> {
|
2021-11-29 12:41:57 +00:00
|
|
|
String canonicalUrl = metadata.canonicalUrl;
|
2021-10-20 01:57:47 +00:00
|
|
|
record Range(long start, long end) {
|
|
|
|
|
|
|
|
long size() {
|
|
|
|
return end - start;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
List<Range> chunks = new ArrayList<>();
|
|
|
|
boolean ranges = metadata.acceptRange && config.downloadThreads() > 1;
|
|
|
|
long chunkSize = ranges ? chunkSizeBytes : metadata.size;
|
|
|
|
for (long start = 0; start < metadata.size; start += chunkSize) {
|
|
|
|
long end = Math.min(start + chunkSize, metadata.size);
|
|
|
|
chunks.add(new Range(start, end));
|
|
|
|
}
|
|
|
|
// create an empty file
|
|
|
|
try {
|
|
|
|
Files.createFile(tmpPath);
|
|
|
|
} catch (IOException e) {
|
|
|
|
return CompletableFuture.failedFuture(new IOException("Failed to create " + resource.output, e));
|
|
|
|
}
|
|
|
|
return WorkerPipeline.start("download-" + resource.id, stats)
|
|
|
|
.readFromTiny("chunks", chunks)
|
|
|
|
.sinkToConsumer("chunk-downloader", Math.min(config.downloadThreads(), chunks.size()), range -> {
|
|
|
|
try (var fileChannel = FileChannel.open(tmpPath, WRITE)) {
|
|
|
|
while (range.size() > 0) {
|
|
|
|
try (
|
2022-03-09 02:08:03 +00:00
|
|
|
var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) :
|
|
|
|
openStream(canonicalUrl);
|
2023-01-30 18:38:09 +00:00
|
|
|
var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress, rateLimiter)
|
2021-10-20 01:57:47 +00:00
|
|
|
) {
|
|
|
|
// ensure this file has been allocated up to the start of this block
|
|
|
|
fileChannel.write(ByteBuffer.allocate(1), range.start);
|
|
|
|
fileChannel.position(range.start);
|
|
|
|
long transferred = fileChannel.transferFrom(input, range.start, range.size());
|
|
|
|
if (transferred == 0) {
|
2021-11-29 12:41:57 +00:00
|
|
|
throw new IOException("Transferred 0 bytes but " + range.size() + " expected: " + canonicalUrl);
|
2021-10-20 01:57:47 +00:00
|
|
|
} else if (transferred != range.size() && !metadata.acceptRange) {
|
|
|
|
throw new IOException(
|
2022-03-09 02:08:03 +00:00
|
|
|
"Transferred " + transferred + " bytes but " + range.size() + " expected: " + canonicalUrl +
|
|
|
|
" and server does not support range requests");
|
2021-10-20 01:57:47 +00:00
|
|
|
}
|
|
|
|
range = new Range(range.start + transferred, range.end);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (IOException e) {
|
|
|
|
throw new UncheckedIOException(e);
|
|
|
|
}
|
|
|
|
}).done();
|
2021-09-10 00:46:20 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
private HttpRequest.Builder newHttpRequest(String url) {
|
|
|
|
return HttpRequest.newBuilder(URI.create(url))
|
2021-10-20 01:57:47 +00:00
|
|
|
.timeout(config.httpTimeout())
|
2021-09-10 00:46:20 +00:00
|
|
|
.header(USER_AGENT, config.httpUserAgent());
|
|
|
|
}
|
|
|
|
|
2022-02-24 01:45:56 +00:00
|
|
|
record ResourceMetadata(Optional<String> redirect, String canonicalUrl, long size, boolean acceptRange) {}
|
2021-10-20 01:57:47 +00:00
|
|
|
|
2022-02-24 01:45:56 +00:00
|
|
|
record ResourceToDownload(
|
2021-10-20 01:57:47 +00:00
|
|
|
String id, String url, Path output, CompletableFuture<ResourceMetadata> metadata, AtomicLong progress
|
|
|
|
) {
|
2021-09-10 00:46:20 +00:00
|
|
|
|
|
|
|
ResourceToDownload(String id, String url, Path output) {
|
2021-10-20 01:57:47 +00:00
|
|
|
this(id, url, output, new CompletableFuture<>(), new AtomicLong(0));
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public Path tmpPath() {
|
|
|
|
return output.resolveSibling(output.getFileName() + "_inprogress");
|
|
|
|
}
|
|
|
|
|
|
|
|
public long bytesDownloaded() {
|
2021-10-20 01:57:47 +00:00
|
|
|
return progress.get();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Wrapper for a {@link ReadableByteChannel} that captures progress information.
|
|
|
|
*/
|
2023-01-30 18:38:09 +00:00
|
|
|
private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress, RateLimiter rateLimiter)
|
|
|
|
implements ReadableByteChannel {
|
2021-10-20 01:57:47 +00:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public int read(ByteBuffer dst) throws IOException {
|
|
|
|
int n = inner.read(dst);
|
|
|
|
if (n > 0) {
|
2023-01-30 18:38:09 +00:00
|
|
|
if (rateLimiter != null) {
|
|
|
|
rateLimiter.acquire(n);
|
|
|
|
}
|
2021-10-20 01:57:47 +00:00
|
|
|
progress.addAndGet(n);
|
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public boolean isOpen() {
|
|
|
|
return inner.isOpen();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public void close() throws IOException {
|
|
|
|
inner.close();
|
2021-09-10 00:46:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|