From 1fa9bc076abeaf2303d807fce297b46a8573d887 Mon Sep 17 00:00:00 2001 From: Taylor Smock Date: Wed, 24 Jun 2020 13:00:28 -0600 Subject: [PATCH] Enable third-party conflation servers Signed-off-by: Taylor Smock --- .classpath | 1 + gradle.properties | 2 +- .../BoundingBoxMapWithAIDownloader.java | 56 ++++++- .../backend/DataConflationSender.java | 143 ++++++++++++++++++ .../MapWithAIConflationCategory.java | 51 +++++++ .../data/mapwithai/MapWithAIInfo.java | 9 ++ .../io/mapwithai/ConflationSourceReader.java | 110 ++++++++++++++ .../io/mapwithai/MapWithAISourceReader.java | 4 +- 8 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java diff --git a/.classpath b/.classpath index 43a5c6b..6a39b0d 100644 --- a/.classpath +++ b/.classpath @@ -19,5 +19,6 @@ + diff --git a/gradle.properties b/gradle.properties index 03f8180..e7be093 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,4 @@ plugin.icon = images/dialogs/mapwithai.svg plugin.link = https://gitlab.com/gokaart/JOSM_MapWithAI plugin.description = Allows the use of MapWithAI data in JOSM (same data as used in RapiD) -plugin.requires = utilsplugin2 +plugin.requires = utilsplugin2;apache-http diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java index b479d26..10e42f1 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java @@ -5,12 +5,19 @@ import static org.openstreetmap.josm.tools.I18n.tr; import java.io.InputStream; import java.net.SocketTimeoutException; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.data.DataSource; import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.gui.MainApplication; import org.openstreetmap.josm.gui.Notification; +import org.openstreetmap.josm.gui.layer.OsmDataLayer; import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; import org.openstreetmap.josm.gui.util.GuiHelper; @@ -21,9 +28,11 @@ import org.openstreetmap.josm.io.OsmApiException; import org.openstreetmap.josm.io.OsmReader; import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin; +import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory; import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo; import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType; import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.Logging; class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader { private final String url; @@ -33,6 +42,7 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader { private final Bounds downloadArea; private final MapWithAIInfo info; + private DataConflationSender dcs; private static final int DEFAULT_TIMEOUT = 50_000; // 50 seconds @@ -56,7 +66,28 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader { public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { long startTime = System.nanoTime(); try { - return super.parseOsm(progressMonitor); + DataSet externalData = super.parseOsm(progressMonitor); + if (!this.info.isConflated() + && !MapWithAIConflationCategory.conflationUrlFor(this.info.getCategory()).isEmpty()) { + if (externalData.getDataSourceBounds().isEmpty()) { + externalData.addDataSource(new DataSource(this.downloadArea, "External Data")); + } + DataSet toConflate = getConflationData(this.downloadArea); + dcs = new DataConflationSender(this.info.getCategory(), toConflate, externalData); + dcs.run(); + try { + DataSet conflatedData = dcs.get(30, TimeUnit.SECONDS); + if (conflatedData != null) { + externalData = conflatedData; + } + } catch (InterruptedException e) { + Logging.error(e); + Thread.currentThread().interrupt(); + } catch (ExecutionException | TimeoutException e) { + Logging.error(e); + } + } + return externalData; } catch (OsmApiException e) { if (!(e.getResponseCode() == 504 && (System.nanoTime() - lastErrorTime) < 120_000_000_000L)) { throw e; @@ -88,6 +119,21 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader { return ds; } + /** + * Get data to send to the conflation server + * + * @param bound The bounds that we are sending to the server + * @return The dataset to send to the server + */ + private static DataSet getConflationData(Bounds bound) { + List layers = MainApplication + .getLayerManager().getLayersOfType(OsmDataLayer.class).stream().filter(l -> l.getDataSet() + .getDataSourceBounds().stream().anyMatch(b -> b.toBBox().bounds(bound.toBBox()))) + .collect(Collectors.toList()); + return layers.stream().max(Comparator.comparingInt(l -> l.getDataSet().allPrimitives().size())) + .map(OsmDataLayer::getDataSet).orElse(null); + } + private static void updateLastErrorTime(long time) { lastErrorTime = time; } @@ -147,4 +193,12 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader { defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo())); request.setHeader("User-Agent", defaultUserAgent.toString()); } + + @Override + public void cancel() { + super.cancel(); + if (dcs != null) { + dcs.cancel(true); + } + } } diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java new file mode 100644 index 0000000..58ac76d --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java @@ -0,0 +1,143 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapwithai.backend; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.io.IllegalDataException; +import org.openstreetmap.josm.io.OsmReader; +import org.openstreetmap.josm.io.OsmWriter; +import org.openstreetmap.josm.io.OsmWriterFactory; +import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory; +import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory; +import org.openstreetmap.josm.tools.Logging; + +/** + * Conflate data with a third party server + * + * @author Taylor Smock + */ +public class DataConflationSender implements RunnableFuture { + + private static final int MAX_POLLS = 100; + private DataSet external; + private DataSet osm; + private MapWithAICategory category; + private DataSet conflatedData; + private CloseableHttpClient client; + private boolean done; + private boolean cancelled; + + /** + * Conflate external data + * + * @param category The category to use to determine the conflation server + * @param openstreetmap The OSM data (may be null -- try to avoid this) + * @param external The data to conflate (may not be null) + */ + public DataConflationSender(MapWithAICategory category, DataSet openstreetmap, DataSet external) { + Objects.requireNonNull(external, tr("We must have data to conflate")); + Objects.requireNonNull(category, tr("We must have a category for the data")); + this.osm = openstreetmap; + this.external = external; + this.category = category; + } + + @Override + public void run() { + String url = MapWithAIConflationCategory.conflationUrlFor(category); + this.client = HttpClients.createDefault(); + try (CloseableHttpClient client = this.client) { + StringWriter output = new StringWriter(); + OsmWriter writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6"); + MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); + if (osm != null) { + writer.write(osm); + multipartEntityBuilder.addTextBody("openstreetmap", output.toString(), ContentType.APPLICATION_XML); + } + // We need to reset the writers to avoid writing previous streams + output = new StringWriter(); + writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6"); + writer.write(external); + multipartEntityBuilder.addTextBody("external", output.toString(), ContentType.APPLICATION_XML); + HttpEntity postData = multipartEntityBuilder.build(); + HttpUriRequest request = RequestBuilder.post(url).setEntity(postData).build(); + + HttpResponse response = client.execute(request); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + conflatedData = OsmReader.parseDataSet(response.getEntity().getContent(), NullProgressMonitor.INSTANCE, + OsmReader.Options.SAVE_ORIGINAL_ID); + } else { + conflatedData = null; + } + this.done = true; + } catch (IOException | UnsupportedOperationException | IllegalDataException e) { + Logging.error(e); + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + try { + client.close(); + } catch (IOException e) { + Logging.error(e); + return false; + } + this.done = true; + this.cancelled = true; + return true; + } + + @Override + public boolean isCancelled() { + return this.cancelled; + } + + @Override + public boolean isDone() { + return this.done; + } + + @Override + public DataSet get() throws InterruptedException, ExecutionException { + while (!isDone()) { + Thread.sleep(100); + } + return this.conflatedData; + } + + @Override + public DataSet get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + long realtime = unit.toMillis(timeout); + long waitTime = realtime > MAX_POLLS ? realtime / MAX_POLLS : 1; + long timeWaited = 0; + while (!isDone()) { + Thread.sleep(waitTime); + timeWaited += waitTime; + } + if (!isDone() && timeWaited > realtime) { + throw new TimeoutException(); + } + return this.conflatedData; + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java new file mode 100644 index 0000000..c898d61 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java @@ -0,0 +1,51 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapwithai.data.mapwithai; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import org.openstreetmap.josm.plugins.mapwithai.io.mapwithai.ConflationSourceReader; +import org.openstreetmap.josm.tools.Logging; + +public class MapWithAIConflationCategory { + private static final Map> CONFLATION_URLS = new EnumMap<>(MapWithAICategory.class); + private static final String EMPTY_URL = ""; + protected static final String DEFAULT_CONFLATION_JSON = "https://gokaart.gitlab.io/JOSM_MapWithAI/json/conflation_servers.json"; + static { + initialize(); + } + + static void initialize() { + try (ConflationSourceReader reader = new ConflationSourceReader(DEFAULT_CONFLATION_JSON)) { + CONFLATION_URLS.putAll(reader.parse()); + } catch (IOException e) { + Logging.error(e); + } + } + + /** + * Get a conflation URL for a specific category + * + * @param category The category for conflation + * @return The URL to use for conflation + */ + public static String conflationUrlFor(MapWithAICategory category) { + return CONFLATION_URLS.getOrDefault(category, Collections.emptyList()).stream().findFirst().orElse(EMPTY_URL); + } + + /** + * Add a conflation URL for a specific category + * + * @param category The category for conflation + * @param url The URL to use for conflation + */ + public static void addConflationUrlFor(MapWithAICategory category, String url) { + Collection list = CONFLATION_URLS.computeIfAbsent(category, i -> new ArrayList<>(1)); + list.add(url); + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java index 8fda7cd..4e9fae7 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java @@ -399,6 +399,15 @@ public class MapWithAIInfo extends this.conflate = conflation; } + /** + * Check if this source is being automatically conflated + * + * @return {@code true} if it should be returned already conflated + */ + public boolean isConflated() { + return this.conflate; + } + /** * Set the URL to use for conflation purposes. * diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java new file mode 100644 index 0000000..53cd09b --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java @@ -0,0 +1,110 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonString; +import javax.json.JsonStructure; +import javax.json.JsonValue; + +import org.openstreetmap.josm.io.CachedFile; +import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.Pair; +import org.openstreetmap.josm.tools.Utils; + +public class ConflationSourceReader implements Closeable { + + private final String source; + private CachedFile cachedFile; + private boolean fastFail; + + /** + * Constructs a {@code ConflationSourceReader} from a given filename, URL or + * internal resource. + * + * @param source can be: + *
    + *
  • relative or absolute file name
    • + *
  • {@code file:///SOME/FILE} the same as above
    • + *
  • {@code http://...} a URL. It will be cached on disk.
    • + *
  • {@code resource://SOME/FILE} file from the classpath + * (usually in the current *.jar)
    • + *
  • {@code josmdir://SOME/FILE} file inside josm user data + * directory (since r7058)
    • + *
  • {@code josmplugindir://SOME/FILE} file inside josm plugin + * directory (since r7834)
    • + *
+ */ + public ConflationSourceReader(String source) { + this.source = source; + } + + /** + * Parses MapWithAI entry sources + * + * @param jsonObject The json of the data sources + * @return The parsed entries + */ + public static Map> parseJson(JsonObject jsonObject) { + return jsonObject.entrySet().stream().flatMap(i -> parse(i).stream()) + .collect(Collectors.groupingBy(p -> p.a, Collectors.mapping(p -> p.b, Collectors.toList()))); + } + + /** + * Parses MapWithAI source. + * + * @return list of source info + * @throws IOException if any I/O error occurs + */ + public Map> parse() throws IOException { + Map> entries = Collections.emptyMap(); + cachedFile = new CachedFile(source); + cachedFile.setFastFail(fastFail); + try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS) + .setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) { + JsonStructure struct = reader.read(); + if (JsonValue.ValueType.OBJECT == struct.getValueType()) { + JsonObject jsonObject = struct.asJsonObject(); + entries = parseJson(jsonObject); + } + return entries; + } + } + + private static List> parse(Map.Entry entry) { + if (JsonValue.ValueType.OBJECT == entry.getValue().getValueType()) { + JsonObject object = entry.getValue().asJsonObject(); + String url = object.getString("url", null); + List categories = object.getJsonArray("categories").getValuesAs(JsonString.class) + .stream().map(JsonString::toString).map(MapWithAICategory::fromString).collect(Collectors.toList()); + return categories.stream().map(c -> new Pair<>(c, url)).collect(Collectors.toList()); + } + return Collections.emptyList(); + } + + /** + * Sets whether opening HTTP connections should fail fast, i.e., whether a + * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. + * + * @param fastFail whether opening HTTP connections should fail fast + * @see CachedFile#setFastFail(boolean) + */ + public void setFastFail(boolean fastFail) { + this.fastFail = fastFail; + } + + @Override + public void close() throws IOException { + Utils.close(cachedFile); + } + +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java index 0366bc2..1dc1cf4 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java @@ -59,8 +59,8 @@ public class MapWithAISourceReader implements Closeable { private static final int COORD_ARRAY_SIZE = 6; /** - * Constructs a {@code ImageryReader} from a given filename, URL or internal - * resource. + * Constructs a {@code MapWithAISourceReader} from a given filename, URL or + * internal resource. * * @param source can be: *