Add `--download-max-bandwidth` option (#467)

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -3,6 +3,7 @@ package com.onthegomap.planetiler.config;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.Storage;
 import com.onthegomap.planetiler.reader.osm.PolyFileReader;
+import com.onthegomap.planetiler.util.Parse;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Path;
@@ -40,6 +41,7 @@ public record PlanetilerConfig(
   int httpRetries,
   long downloadChunkSizeMB,
   int downloadThreads,
+  double downloadMaxBandwidth,
   double minFeatureSizeAtMaxZoom,
   double minFeatureSizeBelowMaxZoom,
   double simplifyToleranceAtMaxZoom,
@@ -148,6 +150,8 @@ public record PlanetilerConfig(
       arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
       arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
       arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
+      Parse.bandwidth(arguments.getString("download_max_bandwidth",
+        "Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc.", "")),
       arguments.getDouble("min_feature_size_at_max_zoom",
         "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming",
         256d / 4096),

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java

@@ -3,6 +3,7 @@ package com.onthegomap.planetiler.util;
 import static com.google.common.net.HttpHeaders.*;
 import static java.nio.file.StandardOpenOption.WRITE;
 
+import com.google.common.util.concurrent.RateLimiter;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.stats.ProgressLoggers;
 import com.onthegomap.planetiler.stats.Stats;
@@ -75,8 +76,10 @@ public class Downloader {
   private final Stats stats;
   private final long chunkSizeBytes;
   private final ResourceUsage diskSpaceCheck = new ResourceUsage("download");
+  private final RateLimiter rateLimiter;
 
   Downloader(PlanetilerConfig config, Stats stats, long chunkSizeBytes) {
+    this.rateLimiter = config.downloadMaxBandwidth() == 0 ? null : RateLimiter.create(config.downloadMaxBandwidth());
     this.chunkSizeBytes = chunkSizeBytes;
     this.config = config;
     this.stats = stats;
@@ -304,7 +307,7 @@ public class Downloader {
               try (
                 var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) :
                   openStream(canonicalUrl);
-                var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress)
+                var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress, rateLimiter)
               ) {
                 // ensure this file has been allocated up to the start of this block
                 fileChannel.write(ByteBuffer.allocate(1), range.start);
@@ -355,12 +358,16 @@ public class Downloader {
   /**
    * Wrapper for a {@link ReadableByteChannel} that captures progress information.
    */
-  private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress) implements ReadableByteChannel {
+  private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress, RateLimiter rateLimiter)
+    implements ReadableByteChannel {
 
     @Override
     public int read(ByteBuffer dst) throws IOException {
       int n = inner.read(dst);
       if (n > 0) {
+        if (rateLimiter != null) {
+          rateLimiter.acquire(n);
+        }
         progress.addAndGet(n);
       }
       return n;

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Parse.java

@@ -19,6 +19,10 @@ public class Parse {
     Pattern.compile(
       "(?<value>-?[\\d.]+)\\s*((?<mi>mi)|(?<m>m|$)|(?<km>km|kilom)|(?<ft>ft|')|(?<in>in|\")|(?<nmi>nmi|international nautical mile|nautical))",
       Pattern.CASE_INSENSITIVE);
+  private static final Pattern NUMBER_WITH_UNIT =
+    Pattern.compile(
+      "(?<value>-?[\\d.]+)\\s*(?<unit>[^.\\d]*)",
+      Pattern.CASE_INSENSITIVE);
   // Ignore warnings about not removing thread local values since planetiler uses dedicated worker threads that release
   // values when a task is finished and are not re-used.
   @SuppressWarnings("java:S5164")
@@ -164,7 +168,7 @@ public class Parse {
   /**
    * Parses {@code tag} as a measure of distance with unit, converted to a round number of meters or {@code null} if
    * invalid.
-   *
+   * <p>
    * See <a href="https://wiki.openstreetmap.org/wiki/Map_features/Units">Map features/Units</a> for the list of
    * supported units.
    */
@@ -205,4 +209,47 @@ public class Parse {
     }
     return null;
   }
+
+  /**
+   * Parses a string containing bandwidth, with units of kbps, mb/s, kib/s, etc.
+   * <p>
+   * Returned value is in units of bytes per second, or 0 if no limit specified.
+   */
+  public static double bandwidth(Object tag) {
+    if (tag != null) {
+      if (tag instanceof Number num) {
+        return num.doubleValue();
+      }
+      var str = tag.toString();
+      var matcher = NUMBER_WITH_UNIT.matcher(str);
+      if (matcher.find()) {
+        try {
+          double value = Double.parseDouble(matcher.group("value"));
+          String unit = matcher.group("unit").toLowerCase(Locale.ROOT).replaceAll("\\s", "");
+          double multiplier = switch (unit) {
+            case "b/s", "" -> 1;
+            case "kb/s" -> 1_000;
+            case "mb/s" -> 1_000_000;
+            case "gb/s" -> 1_000_000_000;
+            case "bps" -> 1d / 8;
+            case "kbps" -> 1_000d / 8;
+            case "mbps" -> 1_000_000d / 8;
+            case "gbps" -> 1_000_000_000d / 8;
+            case "kib/s" -> 1 << 10;
+            case "mib/s" -> 1 << 20;
+            case "gib/s" -> 1 << 30;
+            default -> throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
+          };
+          double result = value * multiplier;
+          if (result < 0) {
+            throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
+          }
+          return result;
+        } catch (NumberFormatException e) {
+          throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
+        }
+      }
+    }
+    return 0;
+  }
 }

planetiler-core/src/test/java/com/onthegomap/planetiler/util/ParseTest.java

@@ -140,4 +140,28 @@ class ParseTest {
   void testParseInvalidJvmSize(String input) {
     assertThrows(IllegalArgumentException.class, () -> Parse.jvmMemoryStringToBytes(input));
   }
+
+  @ParameterizedTest
+  @CsvSource(value = {
+    "0, 0",
+    "1, 1",
+    "999999999999, 999999999999",
+    "2b/s, 2",
+    "2kib/s, 2048",
+    "4mib/s, 4194304",
+    "8GiB/s, 8589934592",
+    "2kb/s, 2000",
+    "4mb/s, 4000000",
+    "8Gb/s, 8000000000",
+    "2bps, 0.25",
+    "8bps, 1",
+    "16bps, 2",
+    "8kbps, 1000",
+    "8mbps, 1000000",
+    "8gbps, 1000000000",
+    "garbage, 0"
+  }, nullValues = {"null"})
+  void testParseBandwidth(String input, double expectedOutput) {
+    assertEquals(expectedOutput, Parse.bandwidth(input));
+  }
 }

planetiler-custommap/planetiler.schema.json

@@ -279,6 +279,9 @@
         "download_threads": {
           "description": "Number of parallel threads to use when downloading each file"
         },
+        "download_max_bandwidth": {
+          "description": "Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc."
+        },
         "min_feature_size_at_max_zoom": {
           "description": "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming"
         },

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java

@@ -204,6 +204,7 @@ public class Contexts {
       argumentValues.put("http_retries", config.httpRetries());
       argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB());
       argumentValues.put("download_threads", config.downloadThreads());
+      argumentValues.put("download_max_bandwidth", config.downloadMaxBandwidth());
       argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom());
       argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
       argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());