mirror
/
MapWithAI
kopia lustrzana https://github.com/JOSM/MapWithAI
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Try to detect when download area is too large (from server response), and try again 

Signed-off-by: Taylor Smock <taylor.smock@kaart.com>
pull/1/head
Taylor Smock 2020-05-07 10:21:48 -06:00
rodzic 8637ca6f96
commit d52ba178f5
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 625F6A74A3E4311A
6 zmienionych plików z 124 dodań i 19 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

49
src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/BoundingBoxMapWithAIDownloader.java
Wyświetl plik

@ -4,12 +4,20 @@ package org.openstreetmap.josm.plugins.mapwithai.backend;
import static org.openstreetmap.josm.tools.I18n.tr;
import java.io.InputStream;
import java.net.SocketTimeoutException;
import java.util.concurrent.TimeUnit;
import org.openstreetmap.josm.data.Bounds;
import org.openstreetmap.josm.data.osm.DataSet;
import org.openstreetmap.josm.gui.MainApplication;
import org.openstreetmap.josm.gui.Notification;
import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
import org.openstreetmap.josm.gui.progress.ProgressMonitor;
import org.openstreetmap.josm.gui.util.GuiHelper;
import org.openstreetmap.josm.io.BoundingBoxDownloader;
import org.openstreetmap.josm.io.IllegalDataException;
import org.openstreetmap.josm.io.OsmApiException;
import org.openstreetmap.josm.io.OsmTransferException;
import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
import org.openstreetmap.josm.tools.HttpClient;
@ -18,6 +26,8 @@ public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
private final String url;
private final boolean crop;
private static long lastErrorTime;
private final Bounds downloadArea;
private MapWithAIInfo info;
@ -37,6 +47,45 @@ public class BoundingBoxMapWithAIDownloader extends BoundingBoxDownloader {
+ (crop ? "&crop_bbox=" + DetectTaskingManagerUtils.getTaskingManagerBBox().toStringCSV(",") : "");
}
@Override
public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
long startTime = System.nanoTime();
try {
return super.parseOsm(progressMonitor);
} catch (OsmApiException e) {
if (!(e.getResponseCode() == 504 && (System.nanoTime() - lastErrorTime) < 120_000_000_000L)) {
throw e;
}
} catch (OsmTransferException e) {
if (e.getCause() instanceof SocketTimeoutException && (System.nanoTime() - startTime) > 30_000_000_000L) {
updateLastErrorTime(System.nanoTime());
Notification note = new Notification();
GuiHelper.runInEDT(() -> note.setContent(tr(
"Attempting to download data in the background. This may fail or succeed in a few minutes.")));
GuiHelper.runInEDT(note::show);
} else {
throw e;
}
}
DataSet ds = new DataSet();
GetDataRunnable runnable = new GetDataRunnable(downloadArea.toBBox(), ds, NullProgressMonitor.INSTANCE);
MainApplication.worker.execute(() -> {
try {
// It seems that the server has issues if I make a request soon
// after the failing request due to a timeout.
TimeUnit.SECONDS.sleep(10);
} catch (InterruptedException e1) {
Thread.currentThread().interrupt();
}
runnable.compute();
});
return ds;
}
private static void updateLastErrorTime(long time) {
lastErrorTime = time;
}
@Override
protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
DataSet ds = OsmReaderCustom.parseDataSet(source, progressMonitor, true);

14
src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/GetDataRunnable.java
Wyświetl plik

@ -56,6 +56,8 @@ public class GetDataRunnable extends RecursiveTask<DataSet> {
private final transient DataSet dataSet;
private final transient ProgressMonitor monitor;
private Integer maximumDimensions;
private static final int MAX_NUMBER_OF_BBOXES_TO_PROCESS = 1;
private static final String SERVER_ID_KEY = "server_id";
@ -86,9 +88,19 @@ public class GetDataRunnable extends RecursiveTask<DataSet> {
this.monitor = Optional.ofNullable(monitor).orElse(NullProgressMonitor.INSTANCE);
}
/**
* Set the maximum download bbox size. Must be called before execution.
*
* @param maximumDimensions The maximum bbox download size
*/
public void setMaximumDimensions(int maximumDimensions) {
this.maximumDimensions = maximumDimensions;
}
@Override
public DataSet compute() {
final List<BBox> bboxes = MapWithAIDataUtils.reduceBBoxSize(bbox);
final List<BBox> bboxes = maximumDimensions == null ? MapWithAIDataUtils.reduceBBoxSize(bbox)
: MapWithAIDataUtils.reduceBBoxSize(bbox, maximumDimensions);
monitor.beginTask(tr("Downloading {0} data ({1} total downloads)", MapWithAIPlugin.NAME, bboxes.size()),
bboxes.size() - 1);
if (!monitor.isCanceled()) {

55
src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtils.java
Wyświetl plik

@ -47,7 +47,7 @@ import org.openstreetmap.josm.tools.Utils;
*/
public final class MapWithAIDataUtils {
/** THe maximum dimensions for MapWithAI data (in kilometers) */
public static final int MAXIMUM_SIDE_DIMENSIONS = 5_000; // RapiD is about 1km, max is 10km, but 10km causes
public static final int MAXIMUM_SIDE_DIMENSIONS = 10_000; // RapiD is about 1km, max is 10km, but 10km causes
// timeouts
private static final int TOO_MANY_BBOXES = 4;
private static ForkJoinPool forkJoinPool;
@ -148,20 +148,45 @@ public final class MapWithAIDataUtils {
* @return A DataSet with data inside the bbox
*/
public static DataSet getData(BBox bbox) {
return getData(Arrays.asList(bbox));
return getData(Arrays.asList(bbox), MAXIMUM_SIDE_DIMENSIONS);
}
/**
* Get a dataset from the API servers using a bbox
*
* @param bbox The bbox from which to get data
* @param maximumDimensions The maximum dimensions to try to download at any one
* time
* @return A DataSet with data inside the bbox
*/
public static DataSet getData(BBox bbox, int maximumDimensions) {
return getData(Arrays.asList(bbox), maximumDimensions);
}
/**
*
* Get a dataset from the API servers using a list bboxes
*
* @param bbox The bboxes from which to get data
* @return A DataSet with data inside the bboxes
*/
public static DataSet getData(List<BBox> bbox) {
return getData(bbox, MAXIMUM_SIDE_DIMENSIONS);
}
/**
* Get a dataset from the API servers using a list bboxes
*
* @param bbox The bboxes from which to get data
* @param maximumDimensions The maximum dimensions to try to download at any one
* time
* @return A DataSet with data inside the bboxes
*/
public static DataSet getData(List<BBox> bbox, int maximumDimensions) {
final DataSet dataSet = new DataSet();
final List<BBox> realBBoxes = bbox.stream().filter(BBox::isValid).distinct().collect(Collectors.toList());
final List<Bounds> realBounds = realBBoxes.stream()
.flatMap(tBBox -> MapWithAIDataUtils.reduceBBoxSize(tBBox).stream())
.flatMap(tBBox -> MapWithAIDataUtils.reduceBBoxSize(tBBox, maximumDimensions).stream())
.map(MapWithAIDataUtils::bboxToBounds).collect(Collectors.toList());
if (!MapWithAILayerInfo.instance.getLayers().isEmpty()) {
if ((realBBoxes.size() < TOO_MANY_BBOXES) || confirmBigDownload(realBBoxes)) {
@ -179,6 +204,9 @@ public final class MapWithAIDataUtils {
}
} catch (OsmTransferException e) {
Logging.error(e);
if (maximumDimensions > MAXIMUM_SIDE_DIMENSIONS / 10) {
dataSet.mergeFrom(getData(bound.toBBox(), maximumDimensions / 2));
}
}
}));
monitor.finishTask();
@ -424,15 +452,16 @@ public final class MapWithAIDataUtils {
/**
* @param bbox The bbox to reduce to a set maximum dimension
* @param maximumDimensions The maximum side dimensions of the bbox
* @return A list of BBoxes that have a dimension no more than
* {@link MAXIMUM_SIDE_DIMENSIONS}
* {@code maximumDimensions}
*/
public static List<BBox> reduceBBoxSize(BBox bbox) {
public static List<BBox> reduceBBoxSize(BBox bbox, int maximumDimensions) {
final List<BBox> returnBounds = new ArrayList<>();
final double width = getWidth(bbox);
final double height = getHeight(bbox);
final Double widthDivisions = width / MAXIMUM_SIDE_DIMENSIONS;
final Double heightDivisions = height / MAXIMUM_SIDE_DIMENSIONS;
final Double widthDivisions = width / maximumDimensions;
final Double heightDivisions = height / maximumDimensions;
final int widthSplits = widthDivisions.intValue() + ((widthDivisions - widthDivisions.intValue()) > 0 ? 1 : 0);
final int heightSplits = heightDivisions.intValue()
+ ((heightDivisions - heightDivisions.intValue()) > 0 ? 1 : 0);
@ -458,8 +487,18 @@ public final class MapWithAIDataUtils {
* {@link MAXIMUM_SIDE_DIMENSIONS}
*/
public static List<BBox> reduceBBoxSize(List<BBox> bboxes) {
return reduceBBoxSize(bboxes, MAXIMUM_SIDE_DIMENSIONS);
}
/**
* @param bboxes The bboxes to reduce to a set maximum dimension
* @param maximumDimensions The maximum width/height dimensions
* @return A list of BBoxes that have a dimension no more than the
* {@code maximumDimensions}
*/
public static List<BBox> reduceBBoxSize(List<BBox> bboxes, int maximumDimensions) {
final List<BBox> returnBBoxes = new ArrayList<>();
bboxes.forEach(bbox -> returnBBoxes.addAll(reduceBBoxSize(bbox)));
bboxes.forEach(bbox -> returnBBoxes.addAll(reduceBBoxSize(bbox, maximumDimensions)));
return returnBBoxes.stream().distinct().collect(Collectors.toList());
}

6
test/unit/org/openstreetmap/josm/plugins/mapwithai/backend/GetDataRunnableTest.java
Wyświetl plik

@ -98,8 +98,10 @@ public class GetDataRunnableTest {
@Test
public void testRegressionTicket46() {
DataSet ds = new DataSet();
new GetDataRunnable(Arrays.asList(new BBox(-5.7400005, 34.4524384, -5.6686014, 34.5513153)), ds, null).fork()
.join();
GetDataRunnable getData = new GetDataRunnable(
Arrays.asList(new BBox(-5.7400005, 34.4524384, -5.6686014, 34.5513153)), ds, null);
getData.setMaximumDimensions(5_000);
getData.fork().join();
assertNotNull(ds);
assertFalse(ds.isEmpty());
assertFalse(ds.allNonDeletedPrimitives().isEmpty());

11
test/unit/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtilsTest.java
Wyświetl plik

@ -175,8 +175,9 @@ public class MapWithAIDataUtilsTest {
final BBox bbox = new BBox(0, 0, 0.0001, 0.0001);
for (Double i : Arrays.asList(0.0001, 0.001, 0.01, 0.1)) {
bbox.add(i, i);
List<BBox> bboxes = MapWithAIDataUtils.reduceBBoxSize(bbox);
assertEquals(getExpectedNumberOfBBoxes(bbox), bboxes.size(), "The bbox should be appropriately reduced");
List<BBox> bboxes = MapWithAIDataUtils.reduceBBoxSize(bbox, 5_000);
assertEquals(getExpectedNumberOfBBoxes(bbox, 5_000), bboxes.size(),
"The bbox should be appropriately reduced");
checkInBBox(bbox, bboxes);
checkBBoxesConnect(bbox, bboxes);
}
@ -192,11 +193,11 @@ public class MapWithAIDataUtilsTest {
.collect(Collectors.toList()).isEmpty());
}
private static int getExpectedNumberOfBBoxes(BBox bbox) {
private static int getExpectedNumberOfBBoxes(BBox bbox, int maximumDimensions) {
double width = MapWithAIDataUtils.getWidth(bbox);
double height = MapWithAIDataUtils.getHeight(bbox);
int widthDivisions = (int) Math.ceil(width / MapWithAIDataUtils.MAXIMUM_SIDE_DIMENSIONS);
int heightDivisions = (int) Math.ceil(height / MapWithAIDataUtils.MAXIMUM_SIDE_DIMENSIONS);
int widthDivisions = (int) Math.ceil(width / maximumDimensions);
int heightDivisions = (int) Math.ceil(height / maximumDimensions);
return widthDivisions * heightDivisions;
}

6
test/unit/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAILayerTest.java
Wyświetl plik

@ -142,8 +142,10 @@ public class MapWithAILayerTest {
public void testSelection() throws InvocationTargetException, InterruptedException {
MapWithAILayer mapWithAILayer = MapWithAIDataUtils.getLayer(true);
DataSet ds = mapWithAILayer.getDataSet();
new GetDataRunnable(Arrays.asList(new BBox(-5.7400005, 34.4524384, -5.6686014, 34.5513153)), ds, null).fork()
.join();
GetDataRunnable getData = new GetDataRunnable(
Arrays.asList(new BBox(-5.7400005, 34.4524384, -5.6686014, 34.5513153)), ds, null);
getData.setMaximumDimensions(5_000);
getData.fork().join();
assertTrue(ds.getSelected().isEmpty());
SwingUtilities.invokeAndWait(() -> ds.setSelected(ds.allNonDeletedCompletePrimitives()));
assertEquals(1, ds.getSelected().size());