diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 6a3cda65..860ab420 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -3,6 +3,7 @@ package com.onthegomap.planetiler.config; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.Storage; import com.onthegomap.planetiler.reader.osm.PolyFileReader; +import com.onthegomap.planetiler.util.Parse; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Path; @@ -40,6 +41,7 @@ public record PlanetilerConfig( int httpRetries, long downloadChunkSizeMB, int downloadThreads, + double downloadMaxBandwidth, double minFeatureSizeAtMaxZoom, double minFeatureSizeBelowMaxZoom, double simplifyToleranceAtMaxZoom, @@ -148,6 +150,8 @@ public record PlanetilerConfig( arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), + Parse.bandwidth(arguments.getString("download_max_bandwidth", + "Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc.", "")), arguments.getDouble("min_feature_size_at_max_zoom", "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming", 256d / 4096), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java index 66c5e746..6deaed26 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java @@ -3,6 +3,7 @@ package com.onthegomap.planetiler.util; import static com.google.common.net.HttpHeaders.*; import static java.nio.file.StandardOpenOption.WRITE; +import com.google.common.util.concurrent.RateLimiter; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.stats.ProgressLoggers; import com.onthegomap.planetiler.stats.Stats; @@ -75,8 +76,10 @@ public class Downloader { private final Stats stats; private final long chunkSizeBytes; private final ResourceUsage diskSpaceCheck = new ResourceUsage("download"); + private final RateLimiter rateLimiter; Downloader(PlanetilerConfig config, Stats stats, long chunkSizeBytes) { + this.rateLimiter = config.downloadMaxBandwidth() == 0 ? null : RateLimiter.create(config.downloadMaxBandwidth()); this.chunkSizeBytes = chunkSizeBytes; this.config = config; this.stats = stats; @@ -304,7 +307,7 @@ public class Downloader { try ( var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) : openStream(canonicalUrl); - var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress) + var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress, rateLimiter) ) { // ensure this file has been allocated up to the start of this block fileChannel.write(ByteBuffer.allocate(1), range.start); @@ -355,12 +358,16 @@ public class Downloader { /** * Wrapper for a {@link ReadableByteChannel} that captures progress information. */ - private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress) implements ReadableByteChannel { + private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress, RateLimiter rateLimiter) + implements ReadableByteChannel { @Override public int read(ByteBuffer dst) throws IOException { int n = inner.read(dst); if (n > 0) { + if (rateLimiter != null) { + rateLimiter.acquire(n); + } progress.addAndGet(n); } return n; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Parse.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Parse.java index 3aa61949..1ddec7c1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Parse.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Parse.java @@ -19,6 +19,10 @@ public class Parse { Pattern.compile( "(?-?[\\d.]+)\\s*((?mi)|(?m|$)|(?km|kilom)|(?ft|')|(?in|\")|(?nmi|international nautical mile|nautical))", Pattern.CASE_INSENSITIVE); + private static final Pattern NUMBER_WITH_UNIT = + Pattern.compile( + "(?-?[\\d.]+)\\s*(?[^.\\d]*)", + Pattern.CASE_INSENSITIVE); // Ignore warnings about not removing thread local values since planetiler uses dedicated worker threads that release // values when a task is finished and are not re-used. @SuppressWarnings("java:S5164") @@ -164,7 +168,7 @@ public class Parse { /** * Parses {@code tag} as a measure of distance with unit, converted to a round number of meters or {@code null} if * invalid. - * + *

* See Map features/Units for the list of * supported units. */ @@ -205,4 +209,47 @@ public class Parse { } return null; } + + /** + * Parses a string containing bandwidth, with units of kbps, mb/s, kib/s, etc. + *

+ * Returned value is in units of bytes per second, or 0 if no limit specified. + */ + public static double bandwidth(Object tag) { + if (tag != null) { + if (tag instanceof Number num) { + return num.doubleValue(); + } + var str = tag.toString(); + var matcher = NUMBER_WITH_UNIT.matcher(str); + if (matcher.find()) { + try { + double value = Double.parseDouble(matcher.group("value")); + String unit = matcher.group("unit").toLowerCase(Locale.ROOT).replaceAll("\\s", ""); + double multiplier = switch (unit) { + case "b/s", "" -> 1; + case "kb/s" -> 1_000; + case "mb/s" -> 1_000_000; + case "gb/s" -> 1_000_000_000; + case "bps" -> 1d / 8; + case "kbps" -> 1_000d / 8; + case "mbps" -> 1_000_000d / 8; + case "gbps" -> 1_000_000_000d / 8; + case "kib/s" -> 1 << 10; + case "mib/s" -> 1 << 20; + case "gib/s" -> 1 << 30; + default -> throw new IllegalArgumentException("Unable to parse bandwidth: " + tag); + }; + double result = value * multiplier; + if (result < 0) { + throw new IllegalArgumentException("Unable to parse bandwidth: " + tag); + } + return result; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unable to parse bandwidth: " + tag); + } + } + } + return 0; + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/ParseTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/ParseTest.java index fedff1bd..d1f83574 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/ParseTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/ParseTest.java @@ -140,4 +140,28 @@ class ParseTest { void testParseInvalidJvmSize(String input) { assertThrows(IllegalArgumentException.class, () -> Parse.jvmMemoryStringToBytes(input)); } + + @ParameterizedTest + @CsvSource(value = { + "0, 0", + "1, 1", + "999999999999, 999999999999", + "2b/s, 2", + "2kib/s, 2048", + "4mib/s, 4194304", + "8GiB/s, 8589934592", + "2kb/s, 2000", + "4mb/s, 4000000", + "8Gb/s, 8000000000", + "2bps, 0.25", + "8bps, 1", + "16bps, 2", + "8kbps, 1000", + "8mbps, 1000000", + "8gbps, 1000000000", + "garbage, 0" + }, nullValues = {"null"}) + void testParseBandwidth(String input, double expectedOutput) { + assertEquals(expectedOutput, Parse.bandwidth(input)); + } } diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index bea2cdd6..99a8c541 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -279,6 +279,9 @@ "download_threads": { "description": "Number of parallel threads to use when downloading each file" }, + "download_max_bandwidth": { + "description": "Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc." + }, "min_feature_size_at_max_zoom": { "description": "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming" }, diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java index 1c15de8d..9cb6e08e 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java @@ -204,6 +204,7 @@ public class Contexts { argumentValues.put("http_retries", config.httpRetries()); argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB()); argumentValues.put("download_threads", config.downloadThreads()); + argumentValues.put("download_max_bandwidth", config.downloadMaxBandwidth()); argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom()); argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom()); argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());