Enable third-party conflation servers

Signed-off-by: Taylor Smock <taylor.smock@kaart.com>

.classpath

@@ -19,5 +19,6 @@
 	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/5"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/JOSM"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/JOSM-utilsplugin2"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/JOSM-apache-http"/>
 	<classpathentry kind="output" path="build/classes/java/main"/>
 </classpath>

gradle.properties

@@ -11,4 +11,4 @@ plugin.icon = images/dialogs/mapwithai.svg
 plugin.link = https://gitlab.com/gokaart/JOSM_MapWithAI
 plugin.description = Allows the use of MapWithAI data in JOSM (same data as used in RapiD)
 
-plugin.requires = utilsplugin2
+plugin.requires = utilsplugin2;apache-http

src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java

@@ -5,12 +5,19 @@ import static org.openstreetmap.josm.tools.I18n.tr;
 
 import java.io.InputStream;
 import java.net.SocketTimeoutException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 
 import org.openstreetmap.josm.data.Bounds;
+import org.openstreetmap.josm.data.DataSource;
 import org.openstreetmap.josm.data.osm.DataSet;
 import org.openstreetmap.josm.gui.MainApplication;
 import org.openstreetmap.josm.gui.Notification;
+import org.openstreetmap.josm.gui.layer.OsmDataLayer;
 import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
 import org.openstreetmap.josm.gui.progress.ProgressMonitor;
 import org.openstreetmap.josm.gui.util.GuiHelper;
@@ -21,9 +28,11 @@ import org.openstreetmap.josm.io.OsmApiException;
 import org.openstreetmap.josm.io.OsmReader;
 import org.openstreetmap.josm.io.OsmTransferException;
 import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
+import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory;
 import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
 import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
 import org.openstreetmap.josm.tools.HttpClient;
+import org.openstreetmap.josm.tools.Logging;
 
 class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
     private final String url;
@@ -33,6 +42,7 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
 
     private final Bounds downloadArea;
     private final MapWithAIInfo info;
+    private DataConflationSender dcs;
 
     private static final int DEFAULT_TIMEOUT = 50_000; // 50 seconds
 
@@ -56,7 +66,28 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
     public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
         long startTime = System.nanoTime();
         try {
-            return super.parseOsm(progressMonitor);
+            DataSet externalData = super.parseOsm(progressMonitor);
+            if (!this.info.isConflated()
+                    && !MapWithAIConflationCategory.conflationUrlFor(this.info.getCategory()).isEmpty()) {
+                if (externalData.getDataSourceBounds().isEmpty()) {
+                    externalData.addDataSource(new DataSource(this.downloadArea, "External Data"));
+                }
+                DataSet toConflate = getConflationData(this.downloadArea);
+                dcs = new DataConflationSender(this.info.getCategory(), toConflate, externalData);
+                dcs.run();
+                try {
+                    DataSet conflatedData = dcs.get(30, TimeUnit.SECONDS);
+                    if (conflatedData != null) {
+                        externalData = conflatedData;
+                    }
+                } catch (InterruptedException e) {
+                    Logging.error(e);
+                    Thread.currentThread().interrupt();
+                } catch (ExecutionException | TimeoutException e) {
+                    Logging.error(e);
+                }
+            }
+            return externalData;
         } catch (OsmApiException e) {
             if (!(e.getResponseCode() == 504 && (System.nanoTime() - lastErrorTime) < 120_000_000_000L)) {
                 throw e;
@@ -88,6 +119,21 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
         return ds;
     }
 
+    /**
+     * Get data to send to the conflation server
+     *
+     * @param bound The bounds that we are sending to the server
+     * @return The dataset to send to the server
+     */
+    private static DataSet getConflationData(Bounds bound) {
+        List<OsmDataLayer> layers = MainApplication
+                .getLayerManager().getLayersOfType(OsmDataLayer.class).stream().filter(l -> l.getDataSet()
+                        .getDataSourceBounds().stream().anyMatch(b -> b.toBBox().bounds(bound.toBBox())))
+                .collect(Collectors.toList());
+        return layers.stream().max(Comparator.comparingInt(l -> l.getDataSet().allPrimitives().size()))
+                .map(OsmDataLayer::getDataSet).orElse(null);
+    }
+
     private static void updateLastErrorTime(long time) {
         lastErrorTime = time;
     }
@@ -147,4 +193,12 @@ class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
         defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo()));
         request.setHeader("User-Agent", defaultUserAgent.toString());
     }
+
+    @Override
+    public void cancel() {
+        super.cancel();
+        if (dcs != null) {
+            dcs.cancel(true);
+        }
+    }
 }

src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/DataConflationSender.java

@@ -0,0 +1,143 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.plugins.mapwithai.backend;
+
+import static org.openstreetmap.josm.tools.I18n.tr;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.openstreetmap.josm.data.osm.DataSet;
+import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
+import org.openstreetmap.josm.io.IllegalDataException;
+import org.openstreetmap.josm.io.OsmReader;
+import org.openstreetmap.josm.io.OsmWriter;
+import org.openstreetmap.josm.io.OsmWriterFactory;
+import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
+import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory;
+import org.openstreetmap.josm.tools.Logging;
+
+/**
+ * Conflate data with a third party server
+ *
+ * @author Taylor Smock
+ */
+public class DataConflationSender implements RunnableFuture<DataSet> {
+
+    private static final int MAX_POLLS = 100;
+    private DataSet external;
+    private DataSet osm;
+    private MapWithAICategory category;
+    private DataSet conflatedData;
+    private CloseableHttpClient client;
+    private boolean done;
+    private boolean cancelled;
+
+    /**
+     * Conflate external data
+     *
+     * @param category      The category to use to determine the conflation server
+     * @param openstreetmap The OSM data (may be null -- try to avoid this)
+     * @param external      The data to conflate (may not be null)
+     */
+    public DataConflationSender(MapWithAICategory category, DataSet openstreetmap, DataSet external) {
+        Objects.requireNonNull(external, tr("We must have data to conflate"));
+        Objects.requireNonNull(category, tr("We must have a category for the data"));
+        this.osm = openstreetmap;
+        this.external = external;
+        this.category = category;
+    }
+
+    @Override
+    public void run() {
+        String url = MapWithAIConflationCategory.conflationUrlFor(category);
+        this.client = HttpClients.createDefault();
+        try (CloseableHttpClient client = this.client) {
+            StringWriter output = new StringWriter();
+            OsmWriter writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6");
+            MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
+            if (osm != null) {
+                writer.write(osm);
+                multipartEntityBuilder.addTextBody("openstreetmap", output.toString(), ContentType.APPLICATION_XML);
+            }
+            // We need to reset the writers to avoid writing previous streams
+            output = new StringWriter();
+            writer = OsmWriterFactory.createOsmWriter(new PrintWriter(output), true, "0.6");
+            writer.write(external);
+            multipartEntityBuilder.addTextBody("external", output.toString(), ContentType.APPLICATION_XML);
+            HttpEntity postData = multipartEntityBuilder.build();
+            HttpUriRequest request = RequestBuilder.post(url).setEntity(postData).build();
+
+            HttpResponse response = client.execute(request);
+            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                conflatedData = OsmReader.parseDataSet(response.getEntity().getContent(), NullProgressMonitor.INSTANCE,
+                        OsmReader.Options.SAVE_ORIGINAL_ID);
+            } else {
+                conflatedData = null;
+            }
+            this.done = true;
+        } catch (IOException | UnsupportedOperationException | IllegalDataException e) {
+            Logging.error(e);
+        }
+    }
+
+    @Override
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        try {
+            client.close();
+        } catch (IOException e) {
+            Logging.error(e);
+            return false;
+        }
+        this.done = true;
+        this.cancelled = true;
+        return true;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return this.cancelled;
+    }
+
+    @Override
+    public boolean isDone() {
+        return this.done;
+    }
+
+    @Override
+    public DataSet get() throws InterruptedException, ExecutionException {
+        while (!isDone()) {
+            Thread.sleep(100);
+        }
+        return this.conflatedData;
+    }
+
+    @Override
+    public DataSet get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+        long realtime = unit.toMillis(timeout);
+        long waitTime = realtime > MAX_POLLS ? realtime / MAX_POLLS : 1;
+        long timeWaited = 0;
+        while (!isDone()) {
+            Thread.sleep(waitTime);
+            timeWaited += waitTime;
+        }
+        if (!isDone() && timeWaited > realtime) {
+            throw new TimeoutException();
+        }
+        return this.conflatedData;
+    }
+}

src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java

@@ -0,0 +1,51 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.plugins.mapwithai.data.mapwithai;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+import org.openstreetmap.josm.plugins.mapwithai.io.mapwithai.ConflationSourceReader;
+import org.openstreetmap.josm.tools.Logging;
+
+public class MapWithAIConflationCategory {
+    private static final Map<MapWithAICategory, List<String>> CONFLATION_URLS = new EnumMap<>(MapWithAICategory.class);
+    private static final String EMPTY_URL = "";
+    protected static final String DEFAULT_CONFLATION_JSON = "https://gokaart.gitlab.io/JOSM_MapWithAI/json/conflation_servers.json";
+    static {
+        initialize();
+    }
+
+    static void initialize() {
+        try (ConflationSourceReader reader = new ConflationSourceReader(DEFAULT_CONFLATION_JSON)) {
+            CONFLATION_URLS.putAll(reader.parse());
+        } catch (IOException e) {
+            Logging.error(e);
+        }
+    }
+
+    /**
+     * Get a conflation URL for a specific category
+     *
+     * @param category The category for conflation
+     * @return The URL to use for conflation
+     */
+    public static String conflationUrlFor(MapWithAICategory category) {
+        return CONFLATION_URLS.getOrDefault(category, Collections.emptyList()).stream().findFirst().orElse(EMPTY_URL);
+    }
+
+    /**
+     * Add a conflation URL for a specific category
+     *
+     * @param category The category for conflation
+     * @param url      The URL to use for conflation
+     */
+    public static void addConflationUrlFor(MapWithAICategory category, String url) {
+        Collection<String> list = CONFLATION_URLS.computeIfAbsent(category, i -> new ArrayList<>(1));
+        list.add(url);
+    }
+}

src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIInfo.java

@@ -399,6 +399,15 @@ public class MapWithAIInfo extends
         this.conflate = conflation;
     }
 
+    /**
+     * Check if this source is being automatically conflated
+     *
+     * @return {@code true} if it should be returned already conflated
+     */
+    public boolean isConflated() {
+        return this.conflate;
+    }
+
     /**
      * Set the URL to use for conflation purposes.
      *

src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java

@@ -0,0 +1,110 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.json.Json;
+import javax.json.JsonObject;
+import javax.json.JsonReader;
+import javax.json.JsonString;
+import javax.json.JsonStructure;
+import javax.json.JsonValue;
+
+import org.openstreetmap.josm.io.CachedFile;
+import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
+import org.openstreetmap.josm.tools.HttpClient;
+import org.openstreetmap.josm.tools.Pair;
+import org.openstreetmap.josm.tools.Utils;
+
+public class ConflationSourceReader implements Closeable {
+
+    private final String source;
+    private CachedFile cachedFile;
+    private boolean fastFail;
+
+    /**
+     * Constructs a {@code ConflationSourceReader} from a given filename, URL or
+     * internal resource.
+     *
+     * @param source can be:
+     *               <ul>
+     *               <li>relative or absolute file name</li>
+     *               <li>{@code file:///SOME/FILE} the same as above</li>
+     *               <li>{@code http://...} a URL. It will be cached on disk.</li>
+     *               <li>{@code resource://SOME/FILE} file from the classpath
+     *               (usually in the current *.jar)</li>
+     *               <li>{@code josmdir://SOME/FILE} file inside josm user data
+     *               directory (since r7058)</li>
+     *               <li>{@code josmplugindir://SOME/FILE} file inside josm plugin
+     *               directory (since r7834)</li>
+     *               </ul>
+     */
+    public ConflationSourceReader(String source) {
+        this.source = source;
+    }
+
+    /**
+     * Parses MapWithAI entry sources
+     *
+     * @param jsonObject The json of the data sources
+     * @return The parsed entries
+     */
+    public static Map<MapWithAICategory, List<String>> parseJson(JsonObject jsonObject) {
+        return jsonObject.entrySet().stream().flatMap(i -> parse(i).stream())
+                .collect(Collectors.groupingBy(p -> p.a, Collectors.mapping(p -> p.b, Collectors.toList())));
+    }
+
+    /**
+     * Parses MapWithAI source.
+     *
+     * @return list of source info
+     * @throws IOException if any I/O error occurs
+     */
+    public Map<MapWithAICategory, List<String>> parse() throws IOException {
+        Map<MapWithAICategory, List<String>> entries = Collections.emptyMap();
+        cachedFile = new CachedFile(source);
+        cachedFile.setFastFail(fastFail);
+        try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS)
+                .setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) {
+            JsonStructure struct = reader.read();
+            if (JsonValue.ValueType.OBJECT == struct.getValueType()) {
+                JsonObject jsonObject = struct.asJsonObject();
+                entries = parseJson(jsonObject);
+            }
+            return entries;
+        }
+    }
+
+    private static List<Pair<MapWithAICategory, String>> parse(Map.Entry<String, JsonValue> entry) {
+        if (JsonValue.ValueType.OBJECT == entry.getValue().getValueType()) {
+            JsonObject object = entry.getValue().asJsonObject();
+            String url = object.getString("url", null);
+            List<MapWithAICategory> categories = object.getJsonArray("categories").getValuesAs(JsonString.class)
+                    .stream().map(JsonString::toString).map(MapWithAICategory::fromString).collect(Collectors.toList());
+            return categories.stream().map(c -> new Pair<>(c, url)).collect(Collectors.toList());
+        }
+        return Collections.emptyList();
+    }
+
+    /**
+     * Sets whether opening HTTP connections should fail fast, i.e., whether a
+     * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
+     *
+     * @param fastFail whether opening HTTP connections should fail fast
+     * @see CachedFile#setFastFail(boolean)
+     */
+    public void setFastFail(boolean fastFail) {
+        this.fastFail = fastFail;
+    }
+
+    @Override
+    public void close() throws IOException {
+        Utils.close(cachedFile);
+    }
+
+}

src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java

@@ -59,8 +59,8 @@ public class MapWithAISourceReader implements Closeable {
     private static final int COORD_ARRAY_SIZE = 6;
 
     /**
-     * Constructs a {@code ImageryReader} from a given filename, URL or internal
+     * Constructs a {@code MapWithAISourceReader} from a given filename, URL or
-     * resource.
+     * internal resource.
      *
      * @param source can be:
      *               <ul>