From 1dc1f18e8ea5b582975a79e2980de3b089b12f13 Mon Sep 17 00:00:00 2001 From: Taylor Smock Date: Mon, 16 May 2022 16:02:29 -0600 Subject: [PATCH] ESRI: Get data in a ForkJoinPool (significant speedup) Signed-off-by: Taylor Smock --- .../data/mapwithai/MapWithAILayerInfo.java | 6 +- .../io/mapwithai/ESRISourceReader.java | 84 ++++++++----------- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java index 8415db0..45effc8 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java @@ -20,7 +20,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Future; +import java.util.concurrent.ForkJoinTask; import java.util.concurrent.RecursiveTask; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -349,7 +349,7 @@ public class MapWithAILayerInfo { private void updateEsriLayers(@Nonnull final Collection layers) { for (MapWithAIInfo layer : layers) { if (MapWithAIType.ESRI == layer.getSourceType()) { - for (Future future : parseEsri(layer)) { + for (ForkJoinTask future : parseEsri(layer)) { try { allDefaultLayers.add(future.get()); } catch (InterruptedException e) { @@ -399,7 +399,7 @@ public class MapWithAILayerInfo { * @param layer The layer to parse * @return The Feature Servers for the ESRI layer */ - private Collection> parseEsri(MapWithAIInfo layer) { + private Collection> parseEsri(MapWithAIInfo layer) { try { return new ESRISourceReader(layer).parse(); } catch (IOException e) { diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java index 5dbffe4..7a3b7d5 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java @@ -23,10 +23,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Future; +import java.util.concurrent.ForkJoinTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; @@ -38,7 +36,6 @@ import org.openstreetmap.josm.data.cache.JCSCacheManager; import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds; import org.openstreetmap.josm.data.preferences.LongProperty; import org.openstreetmap.josm.io.CachedFile; -import org.openstreetmap.josm.plugins.mapwithai.backend.MapWithAIDataUtils; import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory; import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo; import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType; @@ -89,7 +86,7 @@ public class ESRISourceReader { * @return list of source info * @throws IOException if any I/O error occurs */ - public List> parse() throws IOException { + public List> parse() throws IOException { Pattern startReplace = Pattern.compile("\\{start}"); String search = "/search" + JSON_QUERY_PARAM + "&sortField=added&sortOrder=desc&num=" + INITIAL_SEARCH + "&start={start}"; @@ -99,13 +96,11 @@ public class ESRISourceReader { url = url.concat("/"); } - final List> information = Collections.synchronizedList(new ArrayList<>()); + final List> information = new ArrayList<>(); int next = 1; String searchUrl = startReplace.matcher(search).replaceAll(Integer.toString(next)); - final ForkJoinPool forkJoinPool = MapWithAIDataUtils.getForkJoinPool(); - ArrayList> futureList = new ArrayList<>(); while (next != -1) { final String finalUrl = url + "content/groups/" + group + searchUrl; final String jsonString = getJsonString(finalUrl, TimeUnit.SECONDS.toMillis(MIRROR_MAXTIME.get()) / 7, @@ -118,15 +113,11 @@ public class ESRISourceReader { JsonStructure parser = reader.read(); if (parser.getValueType() == JsonValue.ValueType.OBJECT) { JsonObject obj = parser.asJsonObject(); - futureList.ensureCapacity(obj.getInt("total", INITIAL_SEARCH)); next = obj.getInt("nextStart", -1); searchUrl = startReplace.matcher(search).replaceAll(Integer.toString(next)); JsonArray features = obj.getJsonArray("results"); for (JsonObject feature : features.getValuesAs(JsonObject.class)) { - futureList.add(forkJoinPool.submit(() -> { - Future info = parse(feature); - information.add(info); - })); + information.add(parse(feature)); } } } catch (ClassCastException e) { @@ -134,8 +125,9 @@ public class ESRISourceReader { next = -1; } } - for (Future future : futureList) { + for (ForkJoinTask future : information) { try { + future.join(); future.get(1, TimeUnit.MINUTES); } catch (InterruptedException interruptedException) { Logging.warn(interruptedException); @@ -147,17 +139,17 @@ public class ESRISourceReader { return information; } - private Future parse(JsonObject feature) { + private ForkJoinTask parse(JsonObject feature) { // Use the initial esri server information to keep conflation info MapWithAIInfo newInfo = new MapWithAIInfo(source); newInfo.setId(feature.getString("id")); - Future future; + ForkJoinTask future; if (feature.getString("type", "").equals("Feature Service")) { - future = MapWithAIDataUtils.getForkJoinPool() - .submit(() -> newInfo.setUrl(featureService(newInfo, feature.getString("url"))), newInfo); + future = ForkJoinTask + .adapt(() -> newInfo.setUrl(featureService(newInfo, feature.getString("url"))), newInfo).fork(); } else { - future = CompletableFuture.completedFuture(newInfo); newInfo.setUrl(feature.getString("url")); + future = ForkJoinTask.adapt(() -> newInfo).fork(); } newInfo.setName(feature.getString("title", feature.getString("name"))); String[] extent = feature.getJsonArray("extent").getValuesAs(JsonArray.class).stream() @@ -219,38 +211,30 @@ public class ESRISourceReader { CachedFile.cleanup(url); } if (jsonString == null) { - synchronized (SOURCE_CACHE) { - jsonString = SOURCE_CACHE.get(url); - if (jsonString == null) { - HttpClient client = null; - try { - client = HttpClient.create(new URL(url)); - if (fastFail) { - client.setReadTimeout(1000); - } - final HttpClient.Response response = client.connect(); - jsonString = response.fetchContent(); - if (jsonString != null && response.getResponseCode() < 400 - && response.getResponseCode() >= 200) { - final IElementAttributes elementAttributes = SOURCE_CACHE.getDefaultElementAttributes(); - // getExpiration returns milliseconds - final long expirationTime = response.getExpiration(); - if (expirationTime > 0) { - elementAttributes.setMaxLife(response.getExpiration()); - } else { - elementAttributes.setMaxLife(defaultMaxAge); - } - SOURCE_CACHE.put(url, jsonString, elementAttributes); - } else { - jsonString = null; - } - } catch (final IOException e) { - Logging.error(e); - } finally { - if (client != null) { - client.disconnect(); - } + HttpClient client = null; + try { + client = HttpClient.create(new URL(url)); + if (fastFail) { + client.setReadTimeout(1000); + } + final HttpClient.Response response = client.connect(); + jsonString = response.fetchContent(); + if (jsonString != null && response.getResponseCode() < 400 && response.getResponseCode() >= 200) { + // getExpiration returns milliseconds + final long expirationTime = response.getExpiration(); + final IElementAttributes elementAttributes = SOURCE_CACHE.getDefaultElementAttributes(); + if (expirationTime > 0) { + elementAttributes.setMaxLife(response.getExpiration()); + } else { + elementAttributes.setMaxLife(defaultMaxAge); } + SOURCE_CACHE.put(url, jsonString, elementAttributes); + } + } catch (final IOException e) { + Logging.error(e); + } finally { + if (client != null) { + client.disconnect(); } } }