Retry wikidata requests (#115)

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -23,6 +23,7 @@ public record PlanetilerConfig(
   String nodeMapStorage,
   String httpUserAgent,
   Duration httpTimeout,
+  int httpRetries,
   long downloadChunkSizeMB,
   int downloadThreads,
   double minFeatureSizeAtMaxZoom,
@@ -45,6 +46,9 @@ public record PlanetilerConfig(
     if (maxzoom > MAX_MAXZOOM) {
       throw new IllegalArgumentException("Max zoom must be <= " + MAX_MAXZOOM + ", was " + maxzoom);
     }
+    if (httpRetries < 0) {
+      throw new IllegalArgumentException("HTTP Retries must be >= 0, was " + httpRetries);
+    }
   }
 
   public static PlanetilerConfig defaults() {
@@ -74,6 +78,7 @@ public record PlanetilerConfig(
       arguments.getString("http_user_agent", "User-Agent header to set when downloading files over HTTP",
         "Planetiler downloader (https://github.com/onthegomap/planetiler)"),
       arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
+      arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
       arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
       arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
       arguments.getDouble("min_feature_size_at_max_zoom",

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java

@@ -281,7 +281,7 @@ public class Wikidata {
       SELECT ?id ?label where {
         VALUES ?id { %s } ?id (owl:sameAs* / rdfs:label) ?label
       }
-      """.formatted(qidList).replaceAll("\\s+", " ");
+      """.formatted(qidList).replaceAll("\\s+", " ").trim();
 
     HttpRequest request = HttpRequest.newBuilder(URI.create("https://query.wikidata.org/bigdata/namespace/wdq/sparql"))
       .timeout(config.httpTimeout())
@@ -290,10 +290,28 @@ public class Wikidata {
       .header(CONTENT_TYPE, "application/sparql-query")
       .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8))
       .build();
-    InputStream response = client.send(request);
 
-    try (var bis = new BufferedInputStream(response)) {
-      return parseResults(bis);
+    InputStream response = null;
+    for (int i = 0; i <= config.httpRetries() && response == null; i++) {
+      try {
+        response = client.send(request);
+      } catch (IOException e) {
+        boolean lastTry = i == config.httpRetries();
+        if (!lastTry) {
+          LOGGER.warn("sparql query failed, retrying: " + e);
+        } else {
+          LOGGER.error("sparql query failed, exhausted retries: " + e);
+          throw e;
+        }
+      }
+    }
+
+    if (response != null) {
+      try (var bis = new BufferedInputStream(response)) {
+        return parseResults(bis);
+      }
+    } else {
+      throw new IllegalStateException("No response or exception"); // should never happen
     }
   }
 

planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java

@@ -1,10 +1,13 @@
 package com.onthegomap.planetiler.util;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.DynamicTest.dynamicTest;
 
 import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -30,8 +33,48 @@ import org.mockito.Mockito;
 
 public class WikidataTest {
 
-  final PlanetilerConfig config = PlanetilerConfig.defaults();
+  final PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromArgs("--http-retries=1"));
   final Profile profile = new Profile.NullProfile();
+  final String response = """
+    {
+      "head" : {
+        "vars" : [ "id", "label" ]
+      },
+      "results" : {
+        "bindings" : [ {
+          "id" : {
+            "type" : "uri",
+            "value" : "http://www.wikidata.org/entity/Q1"
+          },
+          "label" : {
+            "xml:lang" : "en",
+            "type" : "literal",
+            "value" : "en name"
+          }
+        }, {
+          "id" : {
+            "type" : "uri",
+            "value" : "http://www.wikidata.org/entity/Q1"
+          },
+          "label" : {
+            "xml:lang" : "es",
+            "type" : "literal",
+            "value" : "es name"
+          }
+        }, {
+          "id" : {
+            "type" : "uri",
+            "value" : "http://www.wikidata.org/entity/Q2"
+          },
+          "label" : {
+            "xml:lang" : "es",
+            "type" : "literal",
+            "value" : "es name2"
+          }
+        } ]
+      }
+    }
+    """;
 
   @Test
   public void testWikidataTranslations() {
@@ -56,46 +99,8 @@ public class WikidataTest {
     Wikidata fixture = new Wikidata(writer, client, 2, profile, config);
     fixture.fetch(1L);
     Mockito.verifyNoInteractions(client);
-    Mockito.when(client.send(Mockito.any())).thenReturn(new ByteArrayInputStream("""
-      {
-        "head" : {
-          "vars" : [ "id", "label" ]
-        },
-        "results" : {
-          "bindings" : [ {
-            "id" : {
-              "type" : "uri",
-              "value" : "http://www.wikidata.org/entity/Q1"
-            },
-            "label" : {
-              "xml:lang" : "en",
-              "type" : "literal",
-              "value" : "en name"
-            }
-          }, {
-            "id" : {
-              "type" : "uri",
-              "value" : "http://www.wikidata.org/entity/Q1"
-            },
-            "label" : {
-              "xml:lang" : "es",
-              "type" : "literal",
-              "value" : "es name"
-            }
-          }, {
-            "id" : {
-              "type" : "uri",
-              "value" : "http://www.wikidata.org/entity/Q2"
-            },
-            "label" : {
-              "xml:lang" : "es",
-              "type" : "literal",
-              "value" : "es name2"
-            }
-          } ]
-        }
-      }
-      """.getBytes(StandardCharsets.UTF_8)));
+    Mockito.when(client.send(Mockito.any()))
+      .thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)));
     fixture.fetch(2L);
 
     return List.of(
@@ -133,6 +138,29 @@ public class WikidataTest {
     );
   }
 
+  @Test
+  public void testRetryFailedRequestOnce() throws IOException, InterruptedException {
+    StringWriter writer = new StringWriter();
+    Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
+    Wikidata fixture = new Wikidata(writer, client, 1, profile, config);
+    Mockito.when(client.send(Mockito.any()))
+      // fail once then succeed
+      .thenThrow(IOException.class)
+      .thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)));
+    fixture.fetch(1L);
+    var translations = Wikidata.load(new BufferedReader(new StringReader(writer.toString())));
+    assertEquals(Map.of("en", "en name", "es", "es name"), translations.get(1));
+    assertEquals(Map.of("es", "es name2"), translations.get(2));
+
+    Mockito.reset(client);
+    Mockito.when(client.send(Mockito.any()))
+      // fail all subsequent requests
+      .thenThrow(IOException.class);
+    var outerException = assertThrows(RuntimeException.class, () -> fixture.fetch(2L));
+    var innerException = outerException.getCause();
+    assertInstanceOf(IOException.class, innerException);
+  }
+
   private static void assertEqualsIgnoringWhitespace(String expected, String actual) {
     assertEquals(ignoreWhitespace(expected), ignoreWhitespace(actual));
   }