From d703b626ad8203fe44aaed0a8a0ad1298d349e56 Mon Sep 17 00:00:00 2001
From: Peter Hanecak <115141505+phanecak-maptiler@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:46:17 +0100
Subject: [PATCH] Attempt to fix IOException/GOAWAY happening within
 parseResults() (#818)

---
 .../planetiler/config/PlanetilerConfig.java    |  2 ++
 .../onthegomap/planetiler/util/Wikidata.java   | 18 +++++++++++-------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
index 066afbc8..c505ce08 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
@@ -41,6 +41,7 @@ public record PlanetilerConfig(
   String httpUserAgent,
   Duration httpTimeout,
   int httpRetries,
+  Duration httpRetryWait,
   long downloadChunkSizeMB,
   int downloadThreads,
   double downloadMaxBandwidth,
@@ -166,6 +167,7 @@ public record PlanetilerConfig(
         "Planetiler downloader (https://github.com/onthegomap/planetiler)"),
       arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
       arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
+      arguments.getDuration("http_retry_wait", "How long to wait before retrying HTTP request", "5s"),
       arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
       arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
       Parse.bandwidth(arguments.getString("download_max_bandwidth",
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java
index c3961b00..398713b5 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java
@@ -300,10 +300,15 @@ public class Wikidata {
       .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8))
       .build();
 
-    InputStream response = null;
-    for (int i = 0; i <= config.httpRetries() && response == null; i++) {
+    LongObjectMap<Map<String, String>> result = null;
+    for (int i = 0; i <= config.httpRetries() && result == null; i++) {
       try {
-        response = client.send(request);
+        var response = client.send(request);
+        if (response != null) {
+          try (var bis = new BufferedInputStream(response)) {
+            result = parseResults(bis);
+          }
+        }
       } catch (IOException e) {
         boolean lastTry = i == config.httpRetries();
         if (!lastTry) {
@@ -312,13 +317,12 @@ public class Wikidata {
           LOGGER.error("sparql query failed, exhausted retries: " + e);
           throw e;
         }
+        Thread.sleep(config.httpRetryWait());
       }
     }
 
-    if (response != null) {
-      try (var bis = new BufferedInputStream(response)) {
-        return parseResults(bis);
-      }
+    if (result != null) {
+      return result;
     } else {
       throw new IllegalStateException("No response or exception"); // should never happen
     }