Attempt to fix IOException/GOAWAY happening within parseResults() (#818)

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -41,6 +41,7 @@ public record PlanetilerConfig(
   String httpUserAgent,
   Duration httpTimeout,
   int httpRetries,
+  Duration httpRetryWait,
   long downloadChunkSizeMB,
   int downloadThreads,
   double downloadMaxBandwidth,
@@ -166,6 +167,7 @@ public record PlanetilerConfig(
         "Planetiler downloader (https://github.com/onthegomap/planetiler)"),
       arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
       arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
+      arguments.getDuration("http_retry_wait", "How long to wait before retrying HTTP request", "5s"),
       arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
       arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
       Parse.bandwidth(arguments.getString("download_max_bandwidth",

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java

@@ -300,10 +300,15 @@ public class Wikidata {
       .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8))
       .build();
 
-    InputStream response = null;
+    LongObjectMap<Map<String, String>> result = null;
-    for (int i = 0; i <= config.httpRetries() && response == null; i++) {
+    for (int i = 0; i <= config.httpRetries() && result == null; i++) {
       try {
-        response = client.send(request);
+        var response = client.send(request);
+        if (response != null) {
+          try (var bis = new BufferedInputStream(response)) {
+            result = parseResults(bis);
+          }
+        }
       } catch (IOException e) {
         boolean lastTry = i == config.httpRetries();
         if (!lastTry) {
@@ -312,13 +317,12 @@ public class Wikidata {
           LOGGER.error("sparql query failed, exhausted retries: " + e);
           throw e;
         }
+        Thread.sleep(config.httpRetryWait());
       }
     }
 
-    if (response != null) {
+    if (result != null) {
-      try (var bis = new BufferedInputStream(response)) {
+      return result;
-        return parseResults(bis);
-      }
     } else {
       throw new IllegalStateException("No response or exception"); // should never happen
     }