diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 6085f764..063399ab 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -23,6 +23,7 @@ public record PlanetilerConfig( String nodeMapStorage, String httpUserAgent, Duration httpTimeout, + int httpRetries, long downloadChunkSizeMB, int downloadThreads, double minFeatureSizeAtMaxZoom, @@ -45,6 +46,9 @@ public record PlanetilerConfig( if (maxzoom > MAX_MAXZOOM) { throw new IllegalArgumentException("Max zoom must be <= " + MAX_MAXZOOM + ", was " + maxzoom); } + if (httpRetries < 0) { + throw new IllegalArgumentException("HTTP Retries must be >= 0, was " + httpRetries); + } } public static PlanetilerConfig defaults() { @@ -74,6 +78,7 @@ public record PlanetilerConfig( arguments.getString("http_user_agent", "User-Agent header to set when downloading files over HTTP", "Planetiler downloader (https://github.com/onthegomap/planetiler)"), arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"), + arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), arguments.getDouble("min_feature_size_at_max_zoom", diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index 33183a9e..ae7d1329 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -281,7 +281,7 @@ public class Wikidata { SELECT ?id ?label where { VALUES ?id { %s } ?id (owl:sameAs* / rdfs:label) ?label } - """.formatted(qidList).replaceAll("\\s+", " "); + """.formatted(qidList).replaceAll("\\s+", " ").trim(); HttpRequest request = HttpRequest.newBuilder(URI.create("https://query.wikidata.org/bigdata/namespace/wdq/sparql")) .timeout(config.httpTimeout()) @@ -290,10 +290,28 @@ public class Wikidata { .header(CONTENT_TYPE, "application/sparql-query") .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8)) .build(); - InputStream response = client.send(request); - try (var bis = new BufferedInputStream(response)) { - return parseResults(bis); + InputStream response = null; + for (int i = 0; i <= config.httpRetries() && response == null; i++) { + try { + response = client.send(request); + } catch (IOException e) { + boolean lastTry = i == config.httpRetries(); + if (!lastTry) { + LOGGER.warn("sparql query failed, retrying: " + e); + } else { + LOGGER.error("sparql query failed, exhausted retries: " + e); + throw e; + } + } + } + + if (response != null) { + try (var bis = new BufferedInputStream(response)) { + return parseResults(bis); + } + } else { + throw new IllegalStateException("No response or exception"); // should never happen } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java index c96ef911..45ac17b8 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java @@ -1,10 +1,13 @@ package com.onthegomap.planetiler.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -30,8 +33,48 @@ import org.mockito.Mockito; public class WikidataTest { - final PlanetilerConfig config = PlanetilerConfig.defaults(); + final PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromArgs("--http-retries=1")); final Profile profile = new Profile.NullProfile(); + final String response = """ + { + "head" : { + "vars" : [ "id", "label" ] + }, + "results" : { + "bindings" : [ { + "id" : { + "type" : "uri", + "value" : "http://www.wikidata.org/entity/Q1" + }, + "label" : { + "xml:lang" : "en", + "type" : "literal", + "value" : "en name" + } + }, { + "id" : { + "type" : "uri", + "value" : "http://www.wikidata.org/entity/Q1" + }, + "label" : { + "xml:lang" : "es", + "type" : "literal", + "value" : "es name" + } + }, { + "id" : { + "type" : "uri", + "value" : "http://www.wikidata.org/entity/Q2" + }, + "label" : { + "xml:lang" : "es", + "type" : "literal", + "value" : "es name2" + } + } ] + } + } + """; @Test public void testWikidataTranslations() { @@ -56,46 +99,8 @@ public class WikidataTest { Wikidata fixture = new Wikidata(writer, client, 2, profile, config); fixture.fetch(1L); Mockito.verifyNoInteractions(client); - Mockito.when(client.send(Mockito.any())).thenReturn(new ByteArrayInputStream(""" - { - "head" : { - "vars" : [ "id", "label" ] - }, - "results" : { - "bindings" : [ { - "id" : { - "type" : "uri", - "value" : "http://www.wikidata.org/entity/Q1" - }, - "label" : { - "xml:lang" : "en", - "type" : "literal", - "value" : "en name" - } - }, { - "id" : { - "type" : "uri", - "value" : "http://www.wikidata.org/entity/Q1" - }, - "label" : { - "xml:lang" : "es", - "type" : "literal", - "value" : "es name" - } - }, { - "id" : { - "type" : "uri", - "value" : "http://www.wikidata.org/entity/Q2" - }, - "label" : { - "xml:lang" : "es", - "type" : "literal", - "value" : "es name2" - } - } ] - } - } - """.getBytes(StandardCharsets.UTF_8))); + Mockito.when(client.send(Mockito.any())) + .thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8))); fixture.fetch(2L); return List.of( @@ -133,6 +138,29 @@ public class WikidataTest { ); } + @Test + public void testRetryFailedRequestOnce() throws IOException, InterruptedException { + StringWriter writer = new StringWriter(); + Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); + Wikidata fixture = new Wikidata(writer, client, 1, profile, config); + Mockito.when(client.send(Mockito.any())) + // fail once then succeed + .thenThrow(IOException.class) + .thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8))); + fixture.fetch(1L); + var translations = Wikidata.load(new BufferedReader(new StringReader(writer.toString()))); + assertEquals(Map.of("en", "en name", "es", "es name"), translations.get(1)); + assertEquals(Map.of("es", "es name2"), translations.get(2)); + + Mockito.reset(client); + Mockito.when(client.send(Mockito.any())) + // fail all subsequent requests + .thenThrow(IOException.class); + var outerException = assertThrows(RuntimeException.class, () -> fixture.fetch(2L)); + var innerException = outerException.getCause(); + assertInstanceOf(IOException.class, innerException); + } + private static void assertEqualsIgnoringWhitespace(String expected, String actual) { assertEquals(ignoreWhitespace(expected), ignoreWhitespace(actual)); }