kopia lustrzana https://github.com/JOSM/MapWithAI
Remove custom download code
Signed-off-by: Taylor Smock <taylor.smock@kaart.com>pull/1/head
Taylor Smock
rodzic
78969ec953
commit
d1dce41156
|
|
@ -11,6 +11,7 @@ import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
|||
import org.openstreetmap.josm.io.BoundingBoxDownloader;
|
||||
import org.openstreetmap.josm.io.IllegalDataException;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
|
||||
import org.openstreetmap.josm.tools.HttpClient;
|
||||
|
||||
public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
|
||||
private final String url;
|
||||
|
|
@ -18,6 +19,8 @@ public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
|
|||
|
||||
private final Bounds downloadArea;
|
||||
|
||||
private static final int DEFAULT_TIMEOUT = 50_000; // 50 seconds
|
||||
|
||||
public BoundingBoxMapWithAIDownloader(Bounds downloadArea, String url, boolean crop) {
|
||||
super(downloadArea);
|
||||
this.url = url;
|
||||
|
|
@ -28,7 +31,7 @@ public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
|
|||
@Override
|
||||
protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
|
||||
return url.replace("{bbox}", Double.toString(lon1) + ',' + lat1 + ',' + lon2 + ',' + lat2)
|
||||
+ (crop ? "crop_bbox=" + lon1 + ',' + lat1 + ',' + lon2 + ',' + lat2 : "");
|
||||
+ (crop ? "&crop_bbox=" + DetectTaskingManagerUtils.getTaskingManagerBBox().toStringCSV(",") : "");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -51,4 +54,21 @@ public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
|
|||
protected String getTaskName() {
|
||||
return tr("Contacting {0} Server...", MapWithAIPlugin.NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getBaseUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void adaptRequest(HttpClient request) {
|
||||
final StringBuilder defaultUserAgent = new StringBuilder();
|
||||
request.setReadTimeout(DEFAULT_TIMEOUT);
|
||||
defaultUserAgent.append(request.getHeaders().get("User-Agent"));
|
||||
if (defaultUserAgent.toString().trim().length() == 0) {
|
||||
defaultUserAgent.append("JOSM");
|
||||
}
|
||||
defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo()));
|
||||
request.setHeader("User-Agent", defaultUserAgent.toString());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,10 +4,6 @@ package org.openstreetmap.josm.plugins.mapwithai.backend;
|
|||
import static org.openstreetmap.josm.tools.I18n.tr;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.SocketException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
|
@ -15,14 +11,11 @@ import java.util.Collections;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.RecursiveTask;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.net.ssl.SSLException;
|
||||
|
||||
import org.openstreetmap.josm.actions.MergeNodesAction;
|
||||
import org.openstreetmap.josm.command.Command;
|
||||
import org.openstreetmap.josm.command.DeleteCommand;
|
||||
|
|
@ -40,18 +33,14 @@ import org.openstreetmap.josm.data.osm.Way;
|
|||
import org.openstreetmap.josm.data.osm.WaySegment;
|
||||
import org.openstreetmap.josm.data.projection.ProjectionRegistry;
|
||||
import org.openstreetmap.josm.gui.MainApplication;
|
||||
import org.openstreetmap.josm.gui.Notification;
|
||||
import org.openstreetmap.josm.gui.layer.OsmDataLayer;
|
||||
import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
|
||||
import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
||||
import org.openstreetmap.josm.gui.progress.ProgressMonitor.CancelListener;
|
||||
import org.openstreetmap.josm.io.IllegalDataException;
|
||||
import org.openstreetmap.josm.io.OsmTransferException;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.backend.commands.conflation.DataUrl;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.commands.MergeDuplicateWays;
|
||||
import org.openstreetmap.josm.tools.Geometry;
|
||||
import org.openstreetmap.josm.tools.HttpClient;
|
||||
import org.openstreetmap.josm.tools.HttpClient.Response;
|
||||
import org.openstreetmap.josm.tools.Logging;
|
||||
import org.openstreetmap.josm.tools.Pair;
|
||||
|
||||
|
|
@ -60,10 +49,9 @@ import org.openstreetmap.josm.tools.Pair;
|
|||
*
|
||||
* @author Taylor Smock
|
||||
*/
|
||||
public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelListener {
|
||||
public class GetDataRunnable extends RecursiveTask<DataSet> {
|
||||
private static final long serialVersionUID = 258423685658089715L;
|
||||
private final List<BBox> bbox;
|
||||
private static List<HttpClient> clients;
|
||||
private final transient DataSet dataSet;
|
||||
private final transient ProgressMonitor monitor;
|
||||
|
||||
|
|
@ -98,7 +86,6 @@ public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelLis
|
|||
this.bbox = bbox.stream().distinct().collect(Collectors.toList());
|
||||
this.dataSet = dataSet;
|
||||
this.monitor = Optional.ofNullable(monitor).orElse(NullProgressMonitor.INSTANCE);
|
||||
this.monitor.addCancelListener(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -160,14 +147,16 @@ public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelLis
|
|||
* @param dataSet The dataset with potential duplicate source tags
|
||||
*/
|
||||
public static void removeRedundantSource(DataSet dataSet) {
|
||||
dataSet.getNodes().parallelStream().filter(node -> !node.getReferrers().isEmpty())
|
||||
.filter(node -> node.getKeys().entrySet().parallelStream().map(Entry::getKey)
|
||||
.allMatch(key -> key.contains("source"))
|
||||
&& node.getKeys().entrySet().parallelStream()
|
||||
.allMatch(entry -> node.getReferrers().parallelStream()
|
||||
.anyMatch(parent -> parent.hasTag(entry.getKey(), entry.getValue()))))
|
||||
.forEach(node -> node.getKeys().entrySet().parallelStream().map(Entry::getKey)
|
||||
.filter(key -> key.contains("source")).forEach(node::remove));
|
||||
synchronized (GetDataRunnable.class) {
|
||||
dataSet.getNodes().stream().filter(node -> !node.getReferrers().isEmpty())
|
||||
.filter(node -> node.getKeys().entrySet().parallelStream().map(Entry::getKey)
|
||||
.allMatch(key -> key.contains("source"))
|
||||
&& node.getKeys().entrySet().parallelStream()
|
||||
.allMatch(entry -> node.getReferrers().parallelStream()
|
||||
.anyMatch(parent -> parent.hasTag(entry.getKey(), entry.getValue()))))
|
||||
.forEach(node -> node.getKeys().entrySet().stream().map(Entry::getKey)
|
||||
.filter(key -> key.contains("source")).forEach(node::remove));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -434,15 +423,14 @@ public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelLis
|
|||
|
||||
dataSet.setUploadPolicy(UploadPolicy.DISCOURAGED);
|
||||
|
||||
clients = new ArrayList<>();
|
||||
urlMaps.parallelStream().forEach(map -> {
|
||||
try {
|
||||
final HttpClient client = HttpClient
|
||||
.create(new URL(map.get("url").replace("{bbox}", bbox.toStringCSV(",")).replace("{crop_bbox}",
|
||||
DetectTaskingManagerUtils.getTaskingManagerBBox().toStringCSV(","))));
|
||||
clients.add(client);
|
||||
clientCall(client, dataSet, map.getOrDefault("source", MapWithAIPlugin.NAME), monitor);
|
||||
} catch (final MalformedURLException e1) {
|
||||
Bounds bound = new Bounds(bbox.getBottomRight());
|
||||
bound.extend(bbox.getTopLeft());
|
||||
BoundingBoxMapWithAIDownloader downloader = new BoundingBoxMapWithAIDownloader(bound, map.get("url"),
|
||||
DetectTaskingManagerUtils.hasTaskingManagerLayer());
|
||||
dataSet.mergeFrom(downloader.parseOsm(monitor));
|
||||
} catch (OsmTransferException e1) {
|
||||
Logging.debug(e1);
|
||||
}
|
||||
});
|
||||
|
|
@ -450,68 +438,6 @@ public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelLis
|
|||
return dataSet;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add information to the user agent and then perform the actual internet call
|
||||
*
|
||||
* @param client The HttpClient
|
||||
* @param dataSet The dataset to add data to
|
||||
* @param source The source of the data (added as a tag to "whole" objects)
|
||||
* @param monitor The monitor (so we know when a cancellation has occurred)
|
||||
*/
|
||||
private static void clientCall(HttpClient client, DataSet dataSet, String source, ProgressMonitor monitor) {
|
||||
final StringBuilder defaultUserAgent = new StringBuilder();
|
||||
client.setReadTimeout(DEFAULT_TIMEOUT);
|
||||
defaultUserAgent.append(client.getHeaders().get("User-Agent"));
|
||||
if (defaultUserAgent.toString().trim().length() == 0) {
|
||||
defaultUserAgent.append("JOSM");
|
||||
}
|
||||
defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo()));
|
||||
client.setHeader("User-Agent", defaultUserAgent.toString());
|
||||
if (!monitor.isCanceled()) {
|
||||
clientCallInternet(client, dataSet, source, monitor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add perform an internet request to add data to a dataset
|
||||
*
|
||||
* @param client The HttpClient
|
||||
* @param dataSet The dataset to add data to
|
||||
* @param source The source of the data (added as a tag to "whole" objects)
|
||||
* @param monitor The monitor (so we know when a cancellation has occurred)
|
||||
*/
|
||||
private static void clientCallInternet(HttpClient client, DataSet dataSet, String source, ProgressMonitor monitor) {
|
||||
InputStream inputStream = null;
|
||||
try {
|
||||
Logging.debug("{0}: Getting {1}", MapWithAIPlugin.NAME, client.getURL().toString());
|
||||
final Response response = client.connect();
|
||||
inputStream = response.getContent();
|
||||
final DataSet mergeData = OsmReaderCustom.parseDataSet(inputStream, null, true);
|
||||
addMapWithAISourceTag(mergeData, source);
|
||||
dataSet.mergeFrom(mergeData);
|
||||
response.disconnect();
|
||||
} catch (final SocketException e) {
|
||||
if (!monitor.isCanceled()) {
|
||||
Logging.debug(e);
|
||||
}
|
||||
} catch (final SSLException e) {
|
||||
Logging.debug(e);
|
||||
new Notification(tr("{0}: Bad SSL Certificate: {1}", MapWithAIPlugin.NAME, client.getURL()))
|
||||
.setDuration(Notification.TIME_DEFAULT).show();
|
||||
} catch (UnsupportedOperationException | IllegalDataException | IOException e) {
|
||||
Logging.debug(e);
|
||||
} finally {
|
||||
if (inputStream != null) {
|
||||
try {
|
||||
inputStream.close();
|
||||
} catch (final IOException e) {
|
||||
Logging.debug(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param dataSet The dataset to add the mapwithai source tag to
|
||||
* @param source The source to associate with the data
|
||||
|
|
@ -527,13 +453,6 @@ public class GetDataRunnable extends RecursiveTask<DataSet> implements CancelLis
|
|||
return dataSet;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void operationCanceled() {
|
||||
if (clients != null) {
|
||||
clients.parallelStream().filter(Objects::nonNull).forEach(HttpClient::disconnect);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeObject(java.io.ObjectOutputStream out) throws IOException {
|
||||
out.defaultWriteObject();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,12 +8,14 @@ import java.util.ArrayList;
|
|||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.json.JsonObject;
|
||||
import javax.swing.JOptionPane;
|
||||
import javax.swing.SwingUtilities;
|
||||
|
||||
|
|
@ -34,7 +36,9 @@ import org.openstreetmap.josm.gui.mappaint.MapPaintStyles;
|
|||
import org.openstreetmap.josm.gui.mappaint.StyleSource;
|
||||
import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
|
||||
import org.openstreetmap.josm.gui.progress.swing.PleaseWaitProgressMonitor;
|
||||
import org.openstreetmap.josm.io.OsmTransferException;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.backend.commands.conflation.DataUrl;
|
||||
import org.openstreetmap.josm.plugins.mapwithai.commands.MapWithAIAddCommand;
|
||||
import org.openstreetmap.josm.tools.Logging;
|
||||
import org.openstreetmap.josm.tools.Utils;
|
||||
|
|
@ -157,6 +161,8 @@ public final class MapWithAIDataUtils {
|
|||
public static DataSet getData(List<BBox> bbox) {
|
||||
final DataSet dataSet = new DataSet();
|
||||
final List<BBox> realBBoxes = bbox.stream().filter(BBox::isValid).distinct().collect(Collectors.toList());
|
||||
final List<Bounds> realBounds = realBBoxes.stream().map(MapWithAIDataUtils::bboxToBounds)
|
||||
.collect(Collectors.toList());
|
||||
if (MapWithAIPreferenceHelper.getMapWithAIUrl().parallelStream()
|
||||
.anyMatch(map -> Boolean.valueOf(map.getOrDefault("enabled", "false")))) {
|
||||
if ((realBBoxes.size() < TOO_MANY_BBOXES) || ConditionalOptionPaneUtil.showConfirmationDialog(
|
||||
|
|
@ -165,7 +171,21 @@ public final class MapWithAIDataUtils {
|
|||
realBBoxes.size()),
|
||||
null, JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE, JOptionPane.YES_OPTION)) {
|
||||
final PleaseWaitProgressMonitor monitor = new PleaseWaitProgressMonitor();
|
||||
getForkJoinPool().invoke(new GetDataRunnable(realBBoxes, dataSet, monitor));
|
||||
realBounds.parallelStream()
|
||||
.forEach(bound -> MapWithAIPreferenceHelper.getMapWithAIUrl().parallelStream()
|
||||
.filter(map -> map.containsKey("url")).map(MapWithAIDataUtils::getUrl)
|
||||
.filter(string -> !string.trim().isEmpty()).forEach(url -> {
|
||||
BoundingBoxMapWithAIDownloader downloader = new BoundingBoxMapWithAIDownloader(
|
||||
bound, url, DetectTaskingManagerUtils.hasTaskingManagerLayer());
|
||||
try {
|
||||
DataSet ds = downloader.parseOsm(monitor);
|
||||
synchronized (MapWithAIDataUtils.class) {
|
||||
dataSet.mergeFrom(ds);
|
||||
}
|
||||
} catch (OsmTransferException e) {
|
||||
Logging.error(e);
|
||||
}
|
||||
}));
|
||||
monitor.finishTask();
|
||||
monitor.close();
|
||||
}
|
||||
|
|
@ -185,6 +205,29 @@ public final class MapWithAIDataUtils {
|
|||
return dataSet;
|
||||
}
|
||||
|
||||
private static String getUrl(Map<String, String> urlInformation) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (urlInformation.containsKey("url")) {
|
||||
sb.append(urlInformation.get("url"));
|
||||
if (urlInformation.containsKey("parameters")) {
|
||||
List<String> parameters = DataUrl.readJsonStringArraySimple(urlInformation.get("parameters"))
|
||||
.parallelStream().filter(JsonObject.class::isInstance).map(JsonObject.class::cast)
|
||||
.filter(map -> map.getBoolean("enabled", false)).filter(map -> map.containsKey("parameter"))
|
||||
.map(map -> map.getString("parameter")).collect(Collectors.toList());
|
||||
if (!parameters.isEmpty()) {
|
||||
sb.append('&').append(String.join("&", parameters));
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static Bounds bboxToBounds(BBox bbox) {
|
||||
Bounds bound = new Bounds(bbox.getBottomRight());
|
||||
bound.extend(bbox.getTopLeft());
|
||||
return bound;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The {@link ForkJoinPool} for MapWithAI use.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"id" : "580f7b5b-cb9a-4092-8a39-bf8b714e6efe",
|
||||
"name" : "maps_ml_roads",
|
||||
"request" : {
|
||||
"url" : "/maps/ml_roads?conflate_with_osm=true&theme=ml_road_vector&collaborator=josm&token=ASb3N5o9HbX8QWn8G_NtHIRQaYv3nuG2r7_f3vnGld3KhZNCxg57IsaQyssIaEw5rfRNsPpMwg4TsnrSJtIJms5m&hash=ASawRla3rBcwEjY4HIY&bbox=-108.5715723,39.0734162,-108.5707107,39.0738791&result_type=road_building_vector_xml&crop_bbox=-108.5711561,39.0735205,-108.5708568,39.0736682",
|
||||
"method" : "GET"
|
||||
},
|
||||
"response" : {
|
||||
"status" : 200,
|
||||
"body" : "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<osm generator=\"NS_ROADS_SERVICE\" version=\"0.6\"><bounds maxlat=\"39.0738691\" maxlon=\"-108.57072358101\" minlat=\"39.0734262\" minlon=\"-108.57155941899\"/><node id=\"-2976339053870190188\" lon=\"-108.5708568\" lat=\"39.073597910501\" visible=\"true\"/><way id=\"-4444230471161502072\" orig_id=\"-1203473873173507\" visible=\"true\" action=\"modify\"><tag k=\"highway\" v=\"residential\"/><tag k=\"source\" v=\"digitalglobe\"/><nd ref=\"-710999622096597555\"/><nd ref=\"-373066400259132\"/><nd ref=\"-2976339053870190188\"/></way><node action=\"modify\" id=\"-710999622096597555\" lat=\"39.0736112\" lon=\"-108.5710852\" visible=\"true\"><tag k=\"dupe\" v=\"n6815623346\"/></node><node action=\"modify\" id=\"-373066400259132\" lat=\"39.0735933\" lon=\"-108.5709927\" visible=\"true\"/></osm>\n",
|
||||
"headers" : {
|
||||
"Content-Type" : "text/xml; charset=UTF-8",
|
||||
"Strict-Transport-Security" : "max-age=15552000; preload",
|
||||
"Vary" : "Accept-Encoding",
|
||||
"X-Content-Type-Options" : "nosniff",
|
||||
"X-Frame-Options" : "DENY",
|
||||
"X-XSS-Protection" : "0",
|
||||
"Access-Control-Allow-Origin" : "https://facebook.com",
|
||||
"X-FB-Debug" : "KVSWyv0yjs/bb9oG3tIz+ORdsb/MOp21ZqyqZ4PYVMtcBatm9nD9St1AD+4Y940a7kRKNpcGGjRzWAFf7D8Z2A==",
|
||||
"Date" : "Tue, 17 Dec 2019 21:56:59 GMT",
|
||||
"Alt-Svc" : "h3-24=\":443\"; ma=3600",
|
||||
"Connection" : "keep-alive"
|
||||
}
|
||||
},
|
||||
"uuid" : "580f7b5b-cb9a-4092-8a39-bf8b714e6efe",
|
||||
"persistent" : true,
|
||||
"insertionIndex" : 17
|
||||
}
|
||||
|
|
@ -44,7 +44,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
|||
public class MapWithAIDataUtilsTest {
|
||||
@Rule
|
||||
@SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
|
||||
public JOSMTestRules test = new JOSMTestRules().preferences().main().projection();
|
||||
public JOSMTestRules test = new JOSMTestRules().preferences().main().projection().fakeAPI();
|
||||
|
||||
WireMockServer wireMock = new WireMockServer(options().usingFilesUnderDirectory("test/resources/wiremock"));
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
|||
public class MapWithAILayerTest {
|
||||
@Rule
|
||||
@SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
|
||||
public JOSMTestRules test = new JOSMTestRules().preferences().main().projection();
|
||||
public JOSMTestRules test = new JOSMTestRules().preferences().main().projection().fakeAPI();
|
||||
|
||||
WireMockServer wireMock = new WireMockServer(options().usingFilesUnderDirectory("test/resources/wiremock"));
|
||||
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue