mirror
/
MapWithAI
kopia lustrzana https://github.com/JOSM/MapWithAI
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Enable third-party conflation servers 

Signed-off-by: Taylor Smock <taylor.smock@kaart.com>
pull/1/head
Taylor Smock 2020-06-24 13:00:28 -06:00
rodzic 8469ea09da
commit 1fa9bc076a
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 625F6A74A3E4311A
8 zmienionych plików z 372 dodań i 4 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

1
.classpath
Wyświetl plik

@ -19,5 +19,6 @@
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/5"/>
<classpathentry combineaccessrules="false" kind="src" path="/JOSM"/>
<classpathentry combineaccessrules="false" kind="src" path="/JOSM-utilsplugin2"/>
<classpathentry combineaccessrules="false" kind="src" path="/JOSM-apache-http"/>
<classpathentry kind="output" path="build/classes/java/main"/>
</classpath>

2
gradle.properties
Wyświetl plik

@ -11,4 +11,4 @@ plugin.icon = images/dialogs/mapwithai.svg
plugin.link = https://gitlab.com/gokaart/JOSM_MapWithAI
plugin.description = Allows the use of MapWithAI data in JOSM (same data as used in RapiD)
plugin.requires = utilsplugin2
plugin.requires = utilsplugin2;apache-http

56
src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java
Wyświetl plik

@ -5,12 +5,19 @@ import static org.openstreetmap.josm.tools.I18n.tr;
import java.io.InputStream;
import java.net.SocketTimeoutException;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.openstreetmap.josm.data.Bounds;
import org.openstreetmap.josm.data.DataSource;
import org.openstreetmap.josm.data.osm.DataSet;
import org.openstreetmap.josm.gui.MainApplication;
import org.openstreetmap.josm.gui.Notification;
import org.openstreetmap.josm.gui.layer.OsmDataLayer;
import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
import org.openstreetmap.josm.gui.progress.ProgressMonitor;
import org.openstreetmap.josm.gui.util.GuiHelper;
@ -21,9 +28,11 @@ import org.openstreetmap.josm.io.OsmApiException;
import org.openstreetmap.josm.io.OsmReader;
import org.openstreetmap.josm.io.OsmTransferException;
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Logging;
class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
private final String url;
@ -33,6 +42,7 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
private final Bounds downloadArea;
private final MapWithAIInfo info;
private DataConflationSender dcs;
private static final int DEFAULT_TIMEOUT = 50_000; // 50 seconds
@ -56,7 +66,28 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
long startTime = System.nanoTime();
try {
return super.parseOsm(progressMonitor);
DataSet externalData = super.parseOsm(progressMonitor);
if (!this.info.isConflated()
&& !MapWithAIConflationCategory.conflationUrlFor(this.info.getCategory()).isEmpty()) {
if (externalData.getDataSourceBounds().isEmpty()) {
externalData.addDataSource(new DataSource(this.downloadArea, "External Data"));
}
DataSet toConflate = getConflationData(this.downloadArea);
dcs = new DataConflationSender(this.info.getCategory(), toConflate, externalData);
dcs.run();
try {
DataSet conflatedData = dcs.get(30, TimeUnit.SECONDS);
if (conflatedData != null) {
externalData = conflatedData;
}
} catch (InterruptedException e) {
Logging.error(e);
Thread.currentThread().interrupt();
} catch (ExecutionException | TimeoutException e) {
Logging.error(e);
}
}
return externalData;
} catch (OsmApiException e) {
if (!(e.getResponseCode() == 504 && (System.nanoTime() - lastErrorTime) < 120_000_000_000L)) {
throw e;
@ -88,6 +119,21 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
return ds;
}
/**
* Get data to send to the conflation server
*
* @param bound The bounds that we are sending to the server
* @return The dataset to send to the server
*/
private static DataSet getConflationData(Bounds bound) {
List<OsmDataLayer> layers = MainApplication
.getLayerManager().getLayersOfType(OsmDataLayer.class).stream().filter(l -> l.getDataSet()
.getDataSourceBounds().stream().anyMatch(b -> b.toBBox().bounds(bound.toBBox())))
.collect(Collectors.toList());
return layers.stream().max(Comparator.comparingInt(l -> l.getDataSet().allPrimitives().size()))
.map(OsmDataLayer::getDataSet).orElse(null);
}
private static void updateLastErrorTime(long time) {
lastErrorTime = time;
}
@ -147,4 +193,12 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo()));
request.setHeader("User-Agent", defaultUserAgent.toString());
}
@Override
public void cancel() {
super.cancel();
if (dcs != null) {
dcs.cancel(true);
}
}
}

143
src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java 100644
Wyświetl plik

@ -0,0 +1,143 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.backend;
import static org.openstreetmap.josm.tools.I18n.tr;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.openstreetmap.josm.data.osm.DataSet;
import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
import org.openstreetmap.josm.io.IllegalDataException;
import org.openstreetmap.josm.io.OsmReader;
import org.openstreetmap.josm.io.OsmWriter;
import org.openstreetmap.josm.io.OsmWriterFactory;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory;
import org.openstreetmap.josm.tools.Logging;
/**
* Conflate data with a third party server
*
* @author Taylor Smock
*/
public class DataConflationSender implements RunnableFuture<DataSet> {
private static final int MAX_POLLS = 100;
private DataSet external;
private DataSet osm;
private MapWithAICategory category;
private DataSet conflatedData;
private CloseableHttpClient client;
private boolean done;
private boolean cancelled;
/**
* Conflate external data
*
* @param category The category to use to determine the conflation server
* @param openstreetmap The OSM data (may be null -- try to avoid this)
* @param external The data to conflate (may not be null)
*/
public DataConflationSender(MapWithAICategory category, DataSet openstreetmap, DataSet external) {
Objects.requireNonNull(external, tr("We must have data to conflate"));
Objects.requireNonNull(category, tr("We must have a category for the data"));
this.osm = openstreetmap;
this.external = external;
this.category = category;
}
@Override
public void run() {
String url = MapWithAIConflationCategory.conflationUrlFor(category);
this.client = HttpClients.createDefault();
try (CloseableHttpClient client = this.client) {
StringWriter output = new StringWriter();
OsmWriter writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6");
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
if (osm != null) {
writer.write(osm);
multipartEntityBuilder.addTextBody("openstreetmap", output.toString(), ContentType.APPLICATION_XML);
}
// We need to reset the writers to avoid writing previous streams
output = new StringWriter();
writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6");
writer.write(external);
multipartEntityBuilder.addTextBody("external", output.toString(), ContentType.APPLICATION_XML);
HttpEntity postData = multipartEntityBuilder.build();
HttpUriRequest request = RequestBuilder.post(url).setEntity(postData).build();
HttpResponse response = client.execute(request);
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
conflatedData = OsmReader.parseDataSet(response.getEntity().getContent(), NullProgressMonitor.INSTANCE,
OsmReader.Options.SAVE_ORIGINAL_ID);
} else {
conflatedData = null;
}
this.done = true;
} catch (IOException | UnsupportedOperationException | IllegalDataException e) {
Logging.error(e);
}
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
try {
client.close();
} catch (IOException e) {
Logging.error(e);
return false;
}
this.done = true;
this.cancelled = true;
return true;
}
@Override
public boolean isCancelled() {
return this.cancelled;
}
@Override
public boolean isDone() {
return this.done;
}
@Override
public DataSet get() throws InterruptedException, ExecutionException {
while (!isDone()) {
Thread.sleep(100);
}
return this.conflatedData;
}
@Override
public DataSet get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
long realtime = unit.toMillis(timeout);
long waitTime = realtime > MAX_POLLS ? realtime / MAX_POLLS : 1;
long timeWaited = 0;
while (!isDone()) {
Thread.sleep(waitTime);
timeWaited += waitTime;
}
if (!isDone() && timeWaited > realtime) {
throw new TimeoutException();
}
return this.conflatedData;
}
}

51
src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java 100644
Wyświetl plik

@ -0,0 +1,51 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.data.mapwithai;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import org.openstreetmap.josm.plugins.mapwithai.io.mapwithai.ConflationSourceReader;
import org.openstreetmap.josm.tools.Logging;
public class MapWithAIConflationCategory {
private static final Map<MapWithAICategory, List<String>> CONFLATION_URLS = new EnumMap<>(MapWithAICategory.class);
private static final String EMPTY_URL = "";
protected static final String DEFAULT_CONFLATION_JSON = "https://gokaart.gitlab.io/JOSM_MapWithAI/json/conflation_servers.json";
static {
initialize();
}
static void initialize() {
try (ConflationSourceReader reader = new ConflationSourceReader(DEFAULT_CONFLATION_JSON)) {
CONFLATION_URLS.putAll(reader.parse());
} catch (IOException e) {
Logging.error(e);
}
}
/**
* Get a conflation URL for a specific category
*
* @param category The category for conflation
* @return The URL to use for conflation
*/
public static String conflationUrlFor(MapWithAICategory category) {
return CONFLATION_URLS.getOrDefault(category, Collections.emptyList()).stream().findFirst().orElse(EMPTY_URL);
}
/**
* Add a conflation URL for a specific category
*
* @param category The category for conflation
* @param url The URL to use for conflation
*/
public static void addConflationUrlFor(MapWithAICategory category, String url) {
Collection<String> list = CONFLATION_URLS.computeIfAbsent(category, i -> new ArrayList<>(1));
list.add(url);
}
}

9
src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java
Wyświetl plik

@ -399,6 +399,15 @@ public class MapWithAIInfo extends
this.conflate = conflation;
}
/**
* Check if this source is being automatically conflated
*
* @return {@code true} if it should be returned already conflated
*/
public boolean isConflated() {
return this.conflate;
}
/**
* Set the URL to use for conflation purposes.
*

110
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java 100644
Wyświetl plik

@ -0,0 +1,110 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonString;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import org.openstreetmap.josm.io.CachedFile;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Pair;
import org.openstreetmap.josm.tools.Utils;
public class ConflationSourceReader implements Closeable {
private final String source;
private CachedFile cachedFile;
private boolean fastFail;
/**
* Constructs a {@code ConflationSourceReader} from a given filename, URL or
* internal resource.
*
* @param source can be:
* <ul>
* <li>relative or absolute file name</li>
* <li>{@code file:///SOME/FILE} the same as above</li>
* <li>{@code http://...} a URL. It will be cached on disk.</li>
* <li>{@code resource://SOME/FILE} file from the classpath
* (usually in the current *.jar)</li>
* <li>{@code josmdir://SOME/FILE} file inside josm user data
* directory (since r7058)</li>
* <li>{@code josmplugindir://SOME/FILE} file inside josm plugin
* directory (since r7834)</li>
* </ul>
*/
public ConflationSourceReader(String source) {
this.source = source;
}
/**
* Parses MapWithAI entry sources
*
* @param jsonObject The json of the data sources
* @return The parsed entries
*/
public static Map<MapWithAICategory, List<String>> parseJson(JsonObject jsonObject) {
return jsonObject.entrySet().stream().flatMap(i -> parse(i).stream())
.collect(Collectors.groupingBy(p -> p.a, Collectors.mapping(p -> p.b, Collectors.toList())));
}
/**
* Parses MapWithAI source.
*
* @return list of source info
* @throws IOException if any I/O error occurs
*/
public Map<MapWithAICategory, List<String>> parse() throws IOException {
Map<MapWithAICategory, List<String>> entries = Collections.emptyMap();
cachedFile = new CachedFile(source);
cachedFile.setFastFail(fastFail);
try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS)
.setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) {
JsonStructure struct = reader.read();
if (JsonValue.ValueType.OBJECT == struct.getValueType()) {
JsonObject jsonObject = struct.asJsonObject();
entries = parseJson(jsonObject);
}
return entries;
}
}
private static List<Pair<MapWithAICategory, String>> parse(Map.Entry<String, JsonValue> entry) {
if (JsonValue.ValueType.OBJECT == entry.getValue().getValueType()) {
JsonObject object = entry.getValue().asJsonObject();
String url = object.getString("url", null);
List<MapWithAICategory> categories = object.getJsonArray("categories").getValuesAs(JsonString.class)
.stream().map(JsonString::toString).map(MapWithAICategory::fromString).collect(Collectors.toList());
return categories.stream().map(c -> new Pair<>(c, url)).collect(Collectors.toList());
}
return Collections.emptyList();
}
/**
* Sets whether opening HTTP connections should fail fast, i.e., whether a
* {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
*
* @param fastFail whether opening HTTP connections should fail fast
* @see CachedFile#setFastFail(boolean)
*/
public void setFastFail(boolean fastFail) {
this.fastFail = fastFail;
}
@Override
public void close() throws IOException {
Utils.close(cachedFile);
}
}

4
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java
Wyświetl plik

@ -59,8 +59,8 @@ public class MapWithAISourceReader implements Closeable {
private static final int COORD_ARRAY_SIZE = 6;
/**
* Constructs a {@code ImageryReader} from a given filename, URL or internal
* resource.
* Constructs a {@code MapWithAISourceReader} from a given filename, URL or
* internal resource.
*
* @param source can be:
* <ul>