kopia lustrzana https://github.com/JOSM/MapWithAI
619 wiersze
29 KiB
Java
619 wiersze
29 KiB
Java
// License: GPL. For details, see LICENSE file.
|
|
package org.openstreetmap.josm.plugins.mapwithai.backend;
|
|
|
|
import static org.openstreetmap.josm.tools.I18n.tr;
|
|
|
|
import javax.swing.JOptionPane;
|
|
|
|
import java.io.ByteArrayInputStream;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.io.UncheckedIOException;
|
|
import java.net.MalformedURLException;
|
|
import java.net.SocketTimeoutException;
|
|
import java.net.URI;
|
|
import java.net.URL;
|
|
import java.net.http.HttpRequest;
|
|
import java.net.http.HttpResponse;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.time.Instant;
|
|
import java.util.Arrays;
|
|
import java.util.Collections;
|
|
import java.util.Comparator;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.concurrent.ExecutionException;
|
|
import java.util.concurrent.TimeUnit;
|
|
import java.util.concurrent.TimeoutException;
|
|
import java.util.stream.IntStream;
|
|
import java.util.stream.Stream;
|
|
|
|
import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
|
|
import org.openstreetmap.josm.data.Bounds;
|
|
import org.openstreetmap.josm.data.DataSource;
|
|
import org.openstreetmap.josm.data.imagery.ImageryInfo;
|
|
import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile;
|
|
import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource;
|
|
import org.openstreetmap.josm.data.osm.DataSet;
|
|
import org.openstreetmap.josm.data.osm.IPrimitive;
|
|
import org.openstreetmap.josm.data.osm.Node;
|
|
import org.openstreetmap.josm.data.osm.OsmPrimitive;
|
|
import org.openstreetmap.josm.data.osm.PrimitiveId;
|
|
import org.openstreetmap.josm.data.osm.Relation;
|
|
import org.openstreetmap.josm.data.osm.RelationMember;
|
|
import org.openstreetmap.josm.data.osm.Way;
|
|
import org.openstreetmap.josm.data.vector.VectorNode;
|
|
import org.openstreetmap.josm.data.vector.VectorPrimitive;
|
|
import org.openstreetmap.josm.data.vector.VectorRelation;
|
|
import org.openstreetmap.josm.data.vector.VectorRelationMember;
|
|
import org.openstreetmap.josm.data.vector.VectorWay;
|
|
import org.openstreetmap.josm.gui.MainApplication;
|
|
import org.openstreetmap.josm.gui.Notification;
|
|
import org.openstreetmap.josm.gui.layer.OsmDataLayer;
|
|
import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
|
|
import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
|
import org.openstreetmap.josm.gui.util.GuiHelper;
|
|
import org.openstreetmap.josm.io.BoundingBoxDownloader;
|
|
import org.openstreetmap.josm.io.GeoJSONReader;
|
|
import org.openstreetmap.josm.io.IllegalDataException;
|
|
import org.openstreetmap.josm.io.OsmApiException;
|
|
import org.openstreetmap.josm.io.OsmReader;
|
|
import org.openstreetmap.josm.io.OsmTransferException;
|
|
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
|
|
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIConflationCategory;
|
|
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
|
|
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAILayerInfo;
|
|
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
|
|
import org.openstreetmap.josm.plugins.mapwithai.tools.MapPaintUtils;
|
|
import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo;
|
|
import org.openstreetmap.josm.plugins.pmtiles.gui.layers.PMTilesImageSource;
|
|
import org.openstreetmap.josm.plugins.pmtiles.lib.DirectoryCache;
|
|
import org.openstreetmap.josm.plugins.pmtiles.lib.Header;
|
|
import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles;
|
|
import org.openstreetmap.josm.spi.preferences.Config;
|
|
import org.openstreetmap.josm.tools.HttpClient;
|
|
import org.openstreetmap.josm.tools.JosmRuntimeException;
|
|
import org.openstreetmap.josm.tools.Logging;
|
|
|
|
import jakarta.json.Json;
|
|
import jakarta.json.JsonValue;
|
|
import jakarta.json.stream.JsonParser;
|
|
|
|
/**
|
|
* A bounding box downloader for MapWithAI
|
|
*
|
|
* @author Taylor Smock
|
|
*/
|
|
public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
|
|
private record TileXYZ(int x, int y, int z) {
|
|
/**
|
|
* Checks to see if the given bounds are functionally equal to this tile
|
|
*
|
|
* @param left left
|
|
* @param bottom bottom
|
|
* @param right right
|
|
* @param top top
|
|
*/
|
|
boolean checkBounds(double left, double bottom, double right, double top) {
|
|
final var thisLeft = xToLongitude(this.x, this.z);
|
|
final var thisRight = xToLongitude(this.x + 1, this.z);
|
|
final var thisBottom = yToLatitude(this.y + 1, this.z);
|
|
final var thisTop = yToLatitude(this.y, this.z);
|
|
return equalsEpsilon(thisLeft, left, this.z) && equalsEpsilon(thisRight, right, this.z)
|
|
&& equalsEpsilon(thisBottom, bottom, this.z) && equalsEpsilon(thisTop, top, this.z);
|
|
}
|
|
|
|
private static boolean equalsEpsilon(double first, double second, int z) {
|
|
// 0.1% of tile size is considered to be "equal"
|
|
final var maxDiff = (360 / Math.pow(2, z)) / 1000;
|
|
final var diff = Math.abs(first - second);
|
|
return diff <= maxDiff;
|
|
}
|
|
|
|
private static double xToLongitude(int x, int z) {
|
|
return (x / Math.pow(2, z)) * 360 - 180;
|
|
}
|
|
|
|
private static double yToLatitude(int y, int z) {
|
|
var t = Math.PI - 2 * Math.PI * y / Math.pow(2, z);
|
|
return 180 / Math.PI * Math.atan((Math.exp(t) - Math.exp(-t)) / 2);
|
|
}
|
|
|
|
/**
|
|
* Convert bounds to tiles
|
|
*
|
|
* @param zoom The zoom level to use
|
|
* @param bounds The bounds to convert to tiles
|
|
* @return A stream of tiles for the bounds at the given zoom level
|
|
*/
|
|
private static Stream<TileXYZ> tilesFromBBox(int zoom, Bounds bounds) {
|
|
final var left = bounds.getMinLon();
|
|
final var bottom = bounds.getMinLat();
|
|
final var right = bounds.getMaxLon();
|
|
final var top = bounds.getMaxLat();
|
|
final var tile1 = tileFromLatLonZoom(left, bottom, zoom);
|
|
final var tile2 = tileFromLatLonZoom(right, top, zoom);
|
|
return IntStream.rangeClosed(tile1.x, tile2.x)
|
|
.mapToObj(x -> IntStream.rangeClosed(tile1.y, tile2.y).mapToObj(y -> new TileXYZ(x, y, zoom)))
|
|
.flatMap(stream -> stream);
|
|
}
|
|
|
|
/**
|
|
* Checks to see if the given bounds are functionally equal to this tile
|
|
*
|
|
* @param left left lon
|
|
* @param bottom bottom lat
|
|
* @param right right lon
|
|
* @param top top lat
|
|
*/
|
|
private static TileXYZ tileFromBBox(double left, double bottom, double right, double top) {
|
|
var zoom = 18;
|
|
while (zoom > 0) {
|
|
final var tile1 = tileFromLatLonZoom(left, bottom, zoom);
|
|
final var tile2 = tileFromLatLonZoom(right, top, zoom);
|
|
if (tile1.equals(tile2)) {
|
|
return tile1;
|
|
} else if (tile1.checkBounds(left, bottom, right, top)) {
|
|
return tile1;
|
|
} else if (tile2.checkBounds(left, bottom, right, top)) {
|
|
return tile2;
|
|
// Just in case the coordinates are _barely_ in other tiles and not the "common"
|
|
// tile
|
|
} else if (Math.abs(tile1.x() - tile2.x()) <= 2 && Math.abs(tile1.y() - tile2.y()) <= 2) {
|
|
final var tileT = new TileXYZ((tile1.x() + tile2.x()) / 2, (tile1.y() + tile2.y()) / 2, zoom);
|
|
if (tileT.checkBounds(left, bottom, right, top)) {
|
|
return tileT;
|
|
}
|
|
}
|
|
zoom--;
|
|
}
|
|
return new TileXYZ(0, 0, 0);
|
|
}
|
|
|
|
private static TileXYZ tileFromLatLonZoom(double lon, double lat, int zoom) {
|
|
var xCoordinate = Math.toIntExact(Math.round(Math.floor(Math.pow(2, zoom) * (180 + lon) / 360)));
|
|
var yCoordinate = Math.toIntExact(Math.round(Math.floor(Math.pow(2, zoom)
|
|
* (1 - (Math.log(Math.tan(Math.toRadians(lat)) + 1 / Math.cos(Math.toRadians(lat))) / Math.PI))
|
|
/ 2)));
|
|
return new TileXYZ(xCoordinate, yCoordinate, zoom);
|
|
}
|
|
|
|
/**
|
|
* Extends a bounds object to contain this tile
|
|
*
|
|
* @param currentBounds The bounds to extend
|
|
*/
|
|
private void expandBounds(Bounds currentBounds) {
|
|
final var thisLeft = xToLongitude(this.x, this.z);
|
|
final var thisRight = xToLongitude(this.x + 1, this.z);
|
|
final var thisBottom = yToLatitude(this.y + 1, this.z);
|
|
final var thisTop = yToLatitude(this.y, this.z);
|
|
currentBounds.extend(thisBottom, thisLeft);
|
|
currentBounds.extend(thisTop, thisRight);
|
|
}
|
|
}
|
|
|
|
private final String url;
|
|
private final boolean crop;
|
|
private final int start;
|
|
|
|
private static long lastErrorTime;
|
|
|
|
private final Bounds downloadArea;
|
|
private final MapWithAIInfo info;
|
|
private DataConflationSender dcs;
|
|
|
|
private static final int DEFAULT_TIMEOUT = 50_000; // 50 seconds
|
|
|
|
/**
|
|
* Create a new {@link BoundingBoxMapWithAIDownloader} object
|
|
*
|
|
* @param downloadArea The area to download
|
|
* @param info The info to use to get the url to download
|
|
* @param crop Whether or not to crop the download area
|
|
*/
|
|
public BoundingBoxMapWithAIDownloader(Bounds downloadArea, MapWithAIInfo info, boolean crop) {
|
|
this(downloadArea, info, crop, 0);
|
|
}
|
|
|
|
/**
|
|
* Create a new {@link BoundingBoxMapWithAIDownloader} object
|
|
*
|
|
* @param downloadArea The area to download
|
|
* @param info The info to use to get the url to download
|
|
* @param crop Whether or not to crop the download area
|
|
* @param start The number of objects to skip (Esri only please)
|
|
*/
|
|
private BoundingBoxMapWithAIDownloader(Bounds downloadArea, MapWithAIInfo info, boolean crop, int start) {
|
|
super(downloadArea);
|
|
this.info = info;
|
|
this.url = info.getUrlExpanded();
|
|
this.crop = crop;
|
|
this.downloadArea = downloadArea;
|
|
this.start = start;
|
|
}
|
|
|
|
@Override
|
|
protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
|
|
if (url.contains("{x}") && url.contains("{y}") && url.contains("{z}")) {
|
|
final var tile = TileXYZ.tileFromBBox(lon1, lat1, lon2, lat2);
|
|
return getRequestForTile(tile);
|
|
}
|
|
return url.replace("{bbox}", Double.toString(lon1) + ',' + lat1 + ',' + lon2 + ',' + lat2)
|
|
.replace("{xmin}", Double.toString(lon1)).replace("{ymin}", Double.toString(lat1))
|
|
.replace("{xmax}", Double.toString(lon2)).replace("{ymax}", Double.toString(lat2))
|
|
+ (crop ? "&crop_bbox=" + DetectTaskingManagerUtils.getTaskingManagerBounds().toBBox().toStringCSV(",")
|
|
: "")
|
|
+ (this.info.getSourceType() == MapWithAIType.ESRI_FEATURE_SERVER && !this.info.isConflated()
|
|
? "&resultOffset=" + this.start
|
|
: "");
|
|
}
|
|
|
|
private String getRequestForTile(TileXYZ tile) {
|
|
return url.replace("{x}", Long.toString(tile.x())).replace("{y}", Long.toString(tile.y())).replace("{z}",
|
|
Long.toString(tile.z()));
|
|
}
|
|
|
|
@Override
|
|
public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
|
|
long startTime = System.nanoTime();
|
|
try {
|
|
var externalData = super.parseOsm(progressMonitor);
|
|
// Don't call conflate code unnecessarily
|
|
if ((this.info.getSourceType() != MapWithAIType.ESRI_FEATURE_SERVER || this.start == 0)
|
|
&& Boolean.TRUE.equals(MapWithAIInfo.THIRD_PARTY_CONFLATE.get()) && !this.info.isConflated()
|
|
&& !MapWithAIConflationCategory.conflationUrlFor(this.info.getCategory()).isEmpty()) {
|
|
if (externalData.getDataSourceBounds().isEmpty()) {
|
|
externalData.addDataSource(new DataSource(this.downloadArea, "External Data"));
|
|
}
|
|
final var toConflate = getConflationData(this.downloadArea);
|
|
dcs = new DataConflationSender(this.info.getCategory(), toConflate, externalData);
|
|
dcs.run();
|
|
try {
|
|
final var conflatedData = dcs.get(30, TimeUnit.SECONDS);
|
|
if (conflatedData != null) {
|
|
externalData = conflatedData;
|
|
}
|
|
} catch (InterruptedException e) {
|
|
Logging.error(e);
|
|
Thread.currentThread().interrupt();
|
|
} catch (ExecutionException | TimeoutException e) {
|
|
Logging.error(e);
|
|
}
|
|
}
|
|
MapPaintUtils.addSourcesToPaintStyle(externalData);
|
|
return externalData;
|
|
} catch (OsmApiException e) {
|
|
if (!(e.getResponseCode() == 504 && (System.nanoTime() - lastErrorTime) < 120_000_000_000L)) {
|
|
throw e;
|
|
}
|
|
} catch (OsmTransferException e) {
|
|
if (e.getCause() instanceof SocketTimeoutException && (System.nanoTime() - startTime) > 30_000_000_000L) {
|
|
updateLastErrorTime(System.nanoTime());
|
|
final var note = new Notification();
|
|
GuiHelper.runInEDT(() -> note.setContent(tr(
|
|
"Attempting to download data in the background. This may fail or succeed in a few minutes.")));
|
|
GuiHelper.runInEDT(note::show);
|
|
} else if (e.getCause() instanceof IllegalDataException) {
|
|
final Instant lastUpdated;
|
|
final var now = Instant.now();
|
|
synchronized (BoundingBoxMapWithAIDownloader.class) {
|
|
lastUpdated = Instant.ofEpochSecond(Config.getPref().getLong("mapwithai.layerinfo.lastupdated", 0));
|
|
Config.getPref().putLong("mapwithai.layerinfo.lastupdated", now.getEpochSecond());
|
|
}
|
|
// Only force an update if the last update time is sufficiently old.
|
|
if (now.toEpochMilli() - lastUpdated.toEpochMilli() > TimeUnit.MINUTES.toMillis(10)) {
|
|
MapWithAILayerInfo.getInstance().loadDefaults(true, MapWithAIDataUtils.getForkJoinPool(), false,
|
|
() -> GuiHelper.runInEDT(() -> {
|
|
final var notification = new Notification(tr(
|
|
"MapWithAI layers reloaded. Removing and re-adding the MapWithAI layer may be necessary."));
|
|
notification.setIcon(JOptionPane.INFORMATION_MESSAGE);
|
|
notification.setDuration(Notification.TIME_LONG);
|
|
notification.show();
|
|
}));
|
|
}
|
|
} else {
|
|
throw e;
|
|
}
|
|
}
|
|
// Just in case something happens, try again...
|
|
final var ds = new DataSet();
|
|
final var runnable = new GetDataRunnable(downloadArea, ds, NullProgressMonitor.INSTANCE);
|
|
runnable.setMapWithAIInfo(info);
|
|
MainApplication.worker.execute(() -> {
|
|
try {
|
|
// It seems that the server has issues if I make a request soon
|
|
// after the failing request due to a timeout.
|
|
TimeUnit.SECONDS.sleep(10);
|
|
} catch (InterruptedException e1) {
|
|
Thread.currentThread().interrupt();
|
|
}
|
|
runnable.compute();
|
|
});
|
|
|
|
MapPaintUtils.addSourcesToPaintStyle(ds);
|
|
return ds;
|
|
}
|
|
|
|
/**
|
|
* Get data to send to the conflation server
|
|
*
|
|
* @param bound The bounds that we are sending to the server
|
|
* @return The dataset to send to the server
|
|
*/
|
|
private static DataSet getConflationData(Bounds bound) {
|
|
final var area = DataSource.getDataSourceArea(Collections.singleton(new DataSource(bound, "")));
|
|
if (area != null) {
|
|
final var layers = MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class).stream().filter(
|
|
l -> l.getDataSet().getDataSourceBounds().stream().anyMatch(b -> area.contains(bound.asRect())))
|
|
.toList();
|
|
return layers.stream().max(Comparator.comparingInt(l -> l.getDataSet().allPrimitives().size()))
|
|
.map(OsmDataLayer::getDataSet).orElse(null);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private static void updateLastErrorTime(long time) {
|
|
lastErrorTime = time;
|
|
}
|
|
|
|
@Override
|
|
protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
|
|
DataSet ds;
|
|
final var contentType = this.activeConnection.getResponse().getContentType();
|
|
if (this.info.getSourceType() == MapWithAIType.PMTILES
|
|
|| this.info.getSourceType() == MapWithAIType.MAPBOX_VECTOR_TILE) {
|
|
ds = readMvt(source, progressMonitor);
|
|
} else if (Arrays.asList("text/json", "application/json", "application/geo+json").contains(contentType)
|
|
// Fall back to Esri Feature Server check. They don't always indicate a json
|
|
// return type. :(
|
|
|| (this.info.getSourceType() == MapWithAIType.ESRI_FEATURE_SERVER && !this.info.isConflated())) {
|
|
ds = readJson(source, progressMonitor);
|
|
} else {
|
|
// Fall back to XML parsing
|
|
ds = OsmReader.parseDataSet(source, progressMonitor, OsmReader.Options.CONVERT_UNKNOWN_TO_TAGS,
|
|
OsmReader.Options.SAVE_ORIGINAL_ID);
|
|
}
|
|
if (url != null && info.getUrl() != null && !info.getUrl().trim().isEmpty()) {
|
|
if (info.getSource() != null) {
|
|
GetDataRunnable.addSourceTag(ds, info.getSource());
|
|
} else {
|
|
GetDataRunnable.addMapWithAISourceTag(ds, getMapWithAISourceTag(info));
|
|
}
|
|
}
|
|
GetDataRunnable.cleanup(ds, downloadArea, info);
|
|
return ds;
|
|
}
|
|
|
|
private DataSet readMvt(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
|
|
final DataSet ds;
|
|
final TileSource tileSource;
|
|
final List<TileXYZ> tiles;
|
|
final Header header;
|
|
final DirectoryCache cachedDirectories;
|
|
if (this.info.getSourceType() == MapWithAIType.PMTILES) {
|
|
try {
|
|
header = PMTiles.readHeader(URI.create(this.url));
|
|
final var root = PMTiles.readRootDirectory(header);
|
|
tiles = TileXYZ.tilesFromBBox(header.maxZoom(), this.downloadArea).toList();
|
|
cachedDirectories = new DirectoryCache(root);
|
|
tileSource = new PMTilesImageSource(new PMTilesImageryInfo(header));
|
|
} catch (IOException e) {
|
|
throw new IllegalDataException(e);
|
|
}
|
|
} else {
|
|
header = null;
|
|
cachedDirectories = null;
|
|
// Assume the source is added by the user
|
|
final int zoom;
|
|
if (this.info.getMaxZoom() == 0) {
|
|
var z = 18;
|
|
for (; z > 0; z--) {
|
|
final var tileOptional = TileXYZ.tilesFromBBox(z, this.downloadArea).findFirst();
|
|
if (tileOptional.isPresent()) {
|
|
final var tile = tileOptional.get();
|
|
try {
|
|
final var responseHeader = java.net.http.HttpClient.newBuilder()
|
|
.followRedirects(java.net.http.HttpClient.Redirect.NORMAL).build()
|
|
.send(HttpRequest.newBuilder().uri(URI.create(this.getRequestForTile(tile)))
|
|
.method("HEAD", HttpRequest.BodyPublishers.noBody()).build(),
|
|
HttpResponse.BodyHandlers.discarding());
|
|
if (responseHeader.statusCode() >= 200 && responseHeader.statusCode() < 300) {
|
|
break;
|
|
}
|
|
} catch (IOException e) {
|
|
Logging.trace(e);
|
|
} catch (InterruptedException e) {
|
|
Logging.trace(e);
|
|
Thread.currentThread().interrupt();
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
zoom = z;
|
|
} else {
|
|
zoom = this.info.getMaxZoom();
|
|
}
|
|
tiles = TileXYZ.tilesFromBBox(zoom, this.downloadArea).toList();
|
|
tileSource = new MapboxVectorTileSource(new ImageryInfo(this.url, this.url));
|
|
}
|
|
ds = new DataSet();
|
|
final var currentBounds = new Bounds(this.downloadArea);
|
|
for (TileXYZ tileXYZ : tiles) {
|
|
try {
|
|
final var hilbert = PMTiles.convertToHilbert(tileXYZ.z(), tileXYZ.x(), tileXYZ.y());
|
|
final var data = this.info.getSourceType() == MapWithAIType.PMTILES
|
|
? new ByteArrayInputStream(PMTiles.readData(header, hilbert, cachedDirectories))
|
|
: getInputStream(getRequestForTile(tileXYZ), progressMonitor);
|
|
final var dataSet = loadTile(tileSource, tileXYZ, data);
|
|
ds.mergeFrom(dataSet, progressMonitor);
|
|
tileXYZ.expandBounds(currentBounds);
|
|
} catch (OsmTransferException | IOException e) {
|
|
throw new IllegalDataException(e);
|
|
}
|
|
}
|
|
ds.addDataSource(new DataSource(currentBounds, this.url));
|
|
return ds;
|
|
}
|
|
|
|
private static DataSet loadTile(TileSource tileSource, TileXYZ tileXYZ, InputStream actualSource)
|
|
throws IllegalDataException {
|
|
|
|
final var tile = new MVTTile(tileSource, tileXYZ.x(), tileXYZ.y(), tileXYZ.z());
|
|
try {
|
|
tile.loadImage(actualSource);
|
|
} catch (IOException e) {
|
|
throw new IllegalDataException(e);
|
|
}
|
|
final var ds = new DataSet();
|
|
final var primitiveMap = new HashMap<PrimitiveId, OsmPrimitive>(tile.getData().getAllPrimitives().size());
|
|
for (VectorPrimitive p : tile.getData().getAllPrimitives()) {
|
|
final OsmPrimitive osmPrimitive;
|
|
if (p instanceof VectorNode node) {
|
|
osmPrimitive = new Node(node.getCoor());
|
|
osmPrimitive.putAll(node.getKeys());
|
|
} else if (p instanceof VectorWay way) {
|
|
final var tWay = new Way();
|
|
for (VectorNode node : way.getNodes()) {
|
|
tWay.addNode((Node) primitiveMap.get(node));
|
|
}
|
|
tWay.putAll(way.getKeys());
|
|
osmPrimitive = tWay;
|
|
} else if (p instanceof VectorRelation vectorRelation) {
|
|
final var tRelation = new Relation();
|
|
for (VectorRelationMember member : vectorRelation.getMembers()) {
|
|
tRelation.addMember(new RelationMember(member.getRole(), primitiveMap.get(member.getMember())));
|
|
}
|
|
tRelation.putAll(vectorRelation.getKeys());
|
|
osmPrimitive = tRelation;
|
|
} else {
|
|
throw new IllegalDataException("Unknown vector data type: " + p);
|
|
}
|
|
ds.addPrimitive(osmPrimitive);
|
|
primitiveMap.put(p, osmPrimitive);
|
|
}
|
|
return ds;
|
|
}
|
|
|
|
private DataSet readJson(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
|
|
final DataSet ds;
|
|
// Rather unfortunately, we need to read the entire json in order to figure out
|
|
// if we need to make additional calls
|
|
try (var reader = Json.createReader(source)) {
|
|
final var structure = reader.read();
|
|
try (var bais = new ByteArrayInputStream(structure.toString().getBytes(StandardCharsets.UTF_8))) {
|
|
ds = GeoJSONReader.parseDataSet(bais, progressMonitor);
|
|
} catch (IOException e) {
|
|
throw new UncheckedIOException(e);
|
|
}
|
|
/* We should only call this from the "root" call */
|
|
if (this.start == 0 && structure.getValueType() == JsonValue.ValueType.OBJECT) {
|
|
final var serverObj = structure.asJsonObject();
|
|
final boolean exceededTransferLimit = serverObj.entrySet().stream()
|
|
.filter(entry -> "properties".equals(entry.getKey())
|
|
&& entry.getValue().getValueType() == JsonValue.ValueType.OBJECT)
|
|
.map(Map.Entry::getValue).map(JsonValue::asJsonObject)
|
|
.map(obj -> obj.getBoolean("exceededTransferLimit", false)).findFirst().orElse(false);
|
|
if (exceededTransferLimit && this.info.getSourceType() == MapWithAIType.ESRI_FEATURE_SERVER) {
|
|
final int size = serverObj.getJsonArray("features").size();
|
|
final var other = this.getAdditionalEsriData(progressMonitor,
|
|
this.getRequestForBbox(this.lon1, this.lat1, this.lon2, this.lat2), size);
|
|
ds.mergeFrom(other, progressMonitor.createSubTaskMonitor(0, false));
|
|
}
|
|
}
|
|
}
|
|
if (info.getReplacementTags() != null) {
|
|
GetDataRunnable.replaceKeys(ds, info.getReplacementTags());
|
|
}
|
|
return ds;
|
|
}
|
|
|
|
private DataSet getAdditionalEsriData(ProgressMonitor progressMonitor, String baseUrl, int size) {
|
|
final var returnDs = new DataSet();
|
|
try {
|
|
final var client = HttpClient.create(new URL(baseUrl + "&returnCountOnly=true"));
|
|
int objects = Integer.MIN_VALUE;
|
|
try (var is = client.connect().getContent(); var parser = Json.createParser(is)) {
|
|
while (parser.hasNext()) {
|
|
final var event = parser.next();
|
|
if (event == JsonParser.Event.START_OBJECT) {
|
|
final var objCount = parser.getObjectStream()
|
|
.filter(entry -> "properties".equals(entry.getKey()))
|
|
.map(entry -> entry.getValue().asJsonObject())
|
|
.mapToInt(properties -> properties.getInt("count")).findFirst();
|
|
if (objCount.isPresent()) {
|
|
objects = objCount.getAsInt();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} catch (IOException e) {
|
|
throw new UncheckedIOException(e);
|
|
} finally {
|
|
client.disconnect();
|
|
}
|
|
// Zero indexed. Esri uses 2000 as the limit. 0-1999 is 2000 objects, so we want
|
|
// to start at 2000 for the next round.
|
|
try {
|
|
progressMonitor.beginTask(tr("Downloading additional data"), objects);
|
|
// We have already downloaded some of the objects. Set the ticks.
|
|
progressMonitor.worked(size);
|
|
while (progressMonitor.getTicks() < progressMonitor.getTicksCount() - 1) {
|
|
final var next = new BoundingBoxMapWithAIDownloader(this.downloadArea, this.info, this.crop,
|
|
this.start + size).parseOsm(progressMonitor.createSubTaskMonitor(0, false));
|
|
progressMonitor.worked((int) next.allPrimitives().stream().filter(IPrimitive::isTagged).count());
|
|
returnDs.mergeFrom(next);
|
|
}
|
|
} catch (OsmTransferException e) {
|
|
throw new JosmRuntimeException(e);
|
|
} finally {
|
|
progressMonitor.finishTask();
|
|
}
|
|
} catch (MalformedURLException e) {
|
|
throw new UncheckedIOException(e);
|
|
}
|
|
return returnDs;
|
|
}
|
|
|
|
private static String getMapWithAISourceTag(MapWithAIInfo info) {
|
|
return info.getName() == null ? MapWithAIPlugin.NAME : info.getName();
|
|
}
|
|
|
|
/**
|
|
* Returns the name of the download task to be displayed in the
|
|
* {@link ProgressMonitor}.
|
|
*
|
|
* @return task name
|
|
*/
|
|
@Override
|
|
protected String getTaskName() {
|
|
return tr("Contacting {0} Server...", MapWithAIPlugin.NAME);
|
|
}
|
|
|
|
@Override
|
|
protected String getBaseUrl() {
|
|
return url;
|
|
}
|
|
|
|
@Override
|
|
protected void adaptRequest(HttpClient request) {
|
|
final var defaultUserAgent = new StringBuilder();
|
|
request.setReadTimeout(DEFAULT_TIMEOUT);
|
|
defaultUserAgent.append(request.getHeaders().get("User-Agent"));
|
|
if (defaultUserAgent.toString().trim().length() == 0) {
|
|
defaultUserAgent.append("JOSM");
|
|
}
|
|
defaultUserAgent.append(tr("/ {0} {1}", MapWithAIPlugin.NAME, MapWithAIPlugin.getVersionInfo()));
|
|
request.setHeader("User-Agent", defaultUserAgent.toString());
|
|
}
|
|
|
|
@Override
|
|
public void cancel() {
|
|
super.cancel();
|
|
if (dcs != null) {
|
|
dcs.cancel(true);
|
|
}
|
|
}
|
|
}
|