mirror
/
MapWithAI
kopia lustrzana https://github.com/JOSM/MapWithAI
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

ESRI: Get data in a ForkJoinPool (significant speedup) 

Signed-off-by: Taylor Smock <tsmock@fb.com>
pull/1/head
Taylor Smock 2022-05-16 16:02:29 -06:00
rodzic 5aa3843271
commit 1dc1f18e8e
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 625F6A74A3E4311A
2 zmienionych plików z 37 dodań i 53 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java
Wyświetl plik

@ -20,7 +20,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.RecursiveTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@ -349,7 +349,7 @@ public class MapWithAILayerInfo {
private void updateEsriLayers(@Nonnull final Collection<MapWithAIInfo> layers) {
for (MapWithAIInfo layer : layers) {
if (MapWithAIType.ESRI == layer.getSourceType()) {
for (Future<MapWithAIInfo> future : parseEsri(layer)) {
for (ForkJoinTask<MapWithAIInfo> future : parseEsri(layer)) {
try {
allDefaultLayers.add(future.get());
} catch (InterruptedException e) {
@ -399,7 +399,7 @@ public class MapWithAILayerInfo {
* @param layer The layer to parse
* @return The Feature Servers for the ESRI layer
*/
private Collection<Future<MapWithAIInfo>> parseEsri(MapWithAIInfo layer) {
private Collection<ForkJoinTask<MapWithAIInfo>> parseEsri(MapWithAIInfo layer) {
try {
return new ESRISourceReader(layer).parse();
} catch (IOException e) {

84
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java
Wyświetl plik

@ -23,10 +23,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
@ -38,7 +36,6 @@ import org.openstreetmap.josm.data.cache.JCSCacheManager;
import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
import org.openstreetmap.josm.data.preferences.LongProperty;
import org.openstreetmap.josm.io.CachedFile;
import org.openstreetmap.josm.plugins.mapwithai.backend.MapWithAIDataUtils;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
@ -89,7 +86,7 @@ public class ESRISourceReader {
* @return list of source info
* @throws IOException if any I/O error occurs
*/
public List<Future<MapWithAIInfo>> parse() throws IOException {
public List<ForkJoinTask<MapWithAIInfo>> parse() throws IOException {
Pattern startReplace = Pattern.compile("\\{start}");
String search = "/search" + JSON_QUERY_PARAM + "&sortField=added&sortOrder=desc&num=" + INITIAL_SEARCH
+ "&start={start}";
@ -99,13 +96,11 @@ public class ESRISourceReader {
url = url.concat("/");
}
final List<Future<MapWithAIInfo>> information = Collections.synchronizedList(new ArrayList<>());
final List<ForkJoinTask<MapWithAIInfo>> information = new ArrayList<>();
int next = 1;
String searchUrl = startReplace.matcher(search).replaceAll(Integer.toString(next));
final ForkJoinPool forkJoinPool = MapWithAIDataUtils.getForkJoinPool();
ArrayList<Future<?>> futureList = new ArrayList<>();
while (next != -1) {
final String finalUrl = url + "content/groups/" + group + searchUrl;
final String jsonString = getJsonString(finalUrl, TimeUnit.SECONDS.toMillis(MIRROR_MAXTIME.get()) / 7,
@ -118,15 +113,11 @@ public class ESRISourceReader {
JsonStructure parser = reader.read();
if (parser.getValueType() == JsonValue.ValueType.OBJECT) {
JsonObject obj = parser.asJsonObject();
futureList.ensureCapacity(obj.getInt("total", INITIAL_SEARCH));
next = obj.getInt("nextStart", -1);
searchUrl = startReplace.matcher(search).replaceAll(Integer.toString(next));
JsonArray features = obj.getJsonArray("results");
for (JsonObject feature : features.getValuesAs(JsonObject.class)) {
futureList.add(forkJoinPool.submit(() -> {
Future<MapWithAIInfo> info = parse(feature);
information.add(info);
}));
information.add(parse(feature));
}
}
} catch (ClassCastException e) {
@ -134,8 +125,9 @@ public class ESRISourceReader {
next = -1;
}
}
for (Future<?> future : futureList) {
for (ForkJoinTask<MapWithAIInfo> future : information) {
try {
future.join();
future.get(1, TimeUnit.MINUTES);
} catch (InterruptedException interruptedException) {
Logging.warn(interruptedException);
@ -147,17 +139,17 @@ public class ESRISourceReader {
return information;
}
private Future<MapWithAIInfo> parse(JsonObject feature) {
private ForkJoinTask<MapWithAIInfo> parse(JsonObject feature) {
// Use the initial esri server information to keep conflation info
MapWithAIInfo newInfo = new MapWithAIInfo(source);
newInfo.setId(feature.getString("id"));
Future<MapWithAIInfo> future;
ForkJoinTask<MapWithAIInfo> future;
if (feature.getString("type", "").equals("Feature Service")) {
future = MapWithAIDataUtils.getForkJoinPool()
.submit(() -> newInfo.setUrl(featureService(newInfo, feature.getString("url"))), newInfo);
future = ForkJoinTask
.adapt(() -> newInfo.setUrl(featureService(newInfo, feature.getString("url"))), newInfo).fork();
} else {
future = CompletableFuture.completedFuture(newInfo);
newInfo.setUrl(feature.getString("url"));
future = ForkJoinTask.adapt(() -> newInfo).fork();
}
newInfo.setName(feature.getString("title", feature.getString("name")));
String[] extent = feature.getJsonArray("extent").getValuesAs(JsonArray.class).stream()
@ -219,38 +211,30 @@ public class ESRISourceReader {
CachedFile.cleanup(url);
}
if (jsonString == null) {
synchronized (SOURCE_CACHE) {
jsonString = SOURCE_CACHE.get(url);
if (jsonString == null) {
HttpClient client = null;
try {
client = HttpClient.create(new URL(url));
if (fastFail) {
client.setReadTimeout(1000);
}
final HttpClient.Response response = client.connect();
jsonString = response.fetchContent();
if (jsonString != null && response.getResponseCode() < 400
&& response.getResponseCode() >= 200) {
final IElementAttributes elementAttributes = SOURCE_CACHE.getDefaultElementAttributes();
// getExpiration returns milliseconds
final long expirationTime = response.getExpiration();
if (expirationTime > 0) {
elementAttributes.setMaxLife(response.getExpiration());
} else {
elementAttributes.setMaxLife(defaultMaxAge);
}
SOURCE_CACHE.put(url, jsonString, elementAttributes);
} else {
jsonString = null;
}
} catch (final IOException e) {
Logging.error(e);
} finally {
if (client != null) {
client.disconnect();
}
HttpClient client = null;
try {
client = HttpClient.create(new URL(url));
if (fastFail) {
client.setReadTimeout(1000);
}
final HttpClient.Response response = client.connect();
jsonString = response.fetchContent();
if (jsonString != null && response.getResponseCode() < 400 && response.getResponseCode() >= 200) {
// getExpiration returns milliseconds
final long expirationTime = response.getExpiration();
final IElementAttributes elementAttributes = SOURCE_CACHE.getDefaultElementAttributes();
if (expirationTime > 0) {
elementAttributes.setMaxLife(response.getExpiration());
} else {
elementAttributes.setMaxLife(defaultMaxAge);
}
SOURCE_CACHE.put(url, jsonString, elementAttributes);
}
} catch (final IOException e) {
Logging.error(e);
} finally {
if (client != null) {
client.disconnect();
}
}
}