mirror
/
MapWithAI
kopia lustrzana https://github.com/JOSM/MapWithAI
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

MapWithAI SourceReaders: Create common class for deduplication 

Also ensure that users can ignore cache for MapWithAI sources.

Signed-off-by: Taylor Smock <tsmock@fb.com>
pull/1/head
Taylor Smock 2021-09-30 13:20:03 -06:00
rodzic 073104e869
commit 5cb5262128
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 625F6A74A3E4311A
8 zmienionych plików z 137 dodań i 150 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAIConflationCategory.java
Wyświetl plik

@ -24,7 +24,7 @@ public final class MapWithAIConflationCategory {
static void initialize() {
CONFLATION_URLS.clear();
try (ConflationSourceReader reader = new ConflationSourceReader(conflationJson)) {
CONFLATION_URLS.putAll(reader.parse());
reader.parse().ifPresent(CONFLATION_URLS::putAll);
} catch (IOException e) {
Logging.error(e);
}

9
src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/mapwithai/MapWithAILayerInfo.java
Wyświetl plik

@ -3,6 +3,9 @@ package org.openstreetmap.josm.plugins.mapwithai.data.mapwithai;
import static org.openstreetmap.josm.tools.I18n.tr;
import javax.annotation.Nonnull;
import javax.swing.SwingUtilities;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -21,9 +24,6 @@ import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.swing.SwingUtilities;
import org.openstreetmap.gui.jmapviewer.tilesources.TileSourceInfo;
import org.openstreetmap.josm.actions.ExpertToggleAction;
import org.openstreetmap.josm.data.Preferences;
@ -314,8 +314,9 @@ public class MapWithAILayerInfo {
}
try {
reader = new MapWithAISourceReader(source);
this.reader.setClearCache(this.clearCache);
reader.setFastFail(fastFail);
Collection<MapWithAIInfo> result = reader.parse();
Collection<MapWithAIInfo> result = reader.parse().orElse(Collections.emptyList());
// This is called here to "pre-cache" the layer information, to avoid blocking
// the EDT
this.updateEsriLayers(result);

82
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/CommonSourceReader.java 100644
Wyświetl plik

@ -0,0 +1,82 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import java.io.IOException;
import java.util.Optional;
import org.openstreetmap.josm.io.CachedFile;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Utils;
public abstract class CommonSourceReader<T> implements AutoCloseable {
private final String source;
private CachedFile cachedFile;
private boolean fastFail;
private boolean clearCache;
CommonSourceReader(String source) {
this.source = source;
}
/**
* Parses MapWithAI source information.
*
* @return list of source info
* @throws IOException if any I/O error occurs
*/
public Optional<T> parse() throws IOException {
cachedFile = new CachedFile(source);
cachedFile.setFastFail(fastFail);
if (this.clearCache) {
cachedFile.clear();
}
try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS)
.setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) {
JsonStructure struct = reader.read();
if (JsonValue.ValueType.OBJECT == struct.getValueType()) {
JsonObject jsonObject = struct.asJsonObject();
return Optional.ofNullable(this.parseJson(jsonObject));
}
return Optional.empty();
}
}
/**
* Parses MapWithAI entry sources
*
* @param jsonObject The json of the data sources
* @return The parsed entries
*/
public abstract T parseJson(JsonObject jsonObject);
/**
* Sets whether opening HTTP connections should fail fast, i.e., whether a
* {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
*
* @param fastFail whether opening HTTP connections should fail fast
* @see CachedFile#setFastFail(boolean)
*/
public void setFastFail(boolean fastFail) {
this.fastFail = fastFail;
}
@Override
public void close() throws IOException {
Utils.close(cachedFile);
}
/**
* Indicate if cache should be ignored
*
* @param clearCache {@code true} to ignore cache
*/
public void setClearCache(boolean clearCache) {
this.clearCache = clearCache;
}
}

64
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ConflationSourceReader.java
Wyświetl plik

@ -1,31 +1,21 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import javax.json.JsonObject;
import javax.json.JsonString;
import javax.json.JsonValue;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonString;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import org.openstreetmap.josm.io.CachedFile;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Pair;
import org.openstreetmap.josm.tools.Utils;
public class ConflationSourceReader implements Closeable {
private final String source;
private CachedFile cachedFile;
private boolean fastFail;
public class ConflationSourceReader extends CommonSourceReader<Map<MapWithAICategory, List<String>>>
implements Closeable {
/**
* Constructs a {@code ConflationSourceReader} from a given filename, URL or
@ -45,7 +35,7 @@ public class ConflationSourceReader implements Closeable {
* </ul>
*/
public ConflationSourceReader(String source) {
this.source = source;
super(source);
}
/**
@ -54,32 +44,11 @@ public class ConflationSourceReader implements Closeable {
* @param jsonObject The json of the data sources
* @return The parsed entries
*/
public static Map<MapWithAICategory, List<String>> parseJson(JsonObject jsonObject) {
public Map<MapWithAICategory, List<String>> parseJson(JsonObject jsonObject) {
return jsonObject.entrySet().stream().flatMap(i -> parse(i).stream())
.collect(Collectors.groupingBy(p -> p.a, Collectors.mapping(p -> p.b, Collectors.toList())));
}
/**
* Parses MapWithAI source.
*
* @return list of source info
* @throws IOException if any I/O error occurs
*/
public Map<MapWithAICategory, List<String>> parse() throws IOException {
Map<MapWithAICategory, List<String>> entries = Collections.emptyMap();
cachedFile = new CachedFile(source);
cachedFile.setFastFail(fastFail);
try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS)
.setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) {
JsonStructure struct = reader.read();
if (JsonValue.ValueType.OBJECT == struct.getValueType()) {
JsonObject jsonObject = struct.asJsonObject();
entries = parseJson(jsonObject);
}
return entries;
}
}
private static List<Pair<MapWithAICategory, String>> parse(Map.Entry<String, JsonValue> entry) {
if (JsonValue.ValueType.OBJECT == entry.getValue().getValueType()) {
JsonObject object = entry.getValue().asJsonObject();
@ -90,21 +59,4 @@ public class ConflationSourceReader implements Closeable {
}
return Collections.emptyList();
}
/**
* Sets whether opening HTTP connections should fail fast, i.e., whether a
* {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
*
* @param fastFail whether opening HTTP connections should fail fast
* @see CachedFile#setFastFail(boolean)
*/
public void setFastFail(boolean fastFail) {
this.fastFail = fastFail;
}
@Override
public void close() throws IOException {
Utils.close(cachedFile);
}
}

35
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/ESRISourceReader.java
Wyświetl plik

@ -1,6 +1,18 @@
// License: GPL. For details, see LICENSE file.
package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonNumber;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonString;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import javax.json.stream.JsonParsingException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
@ -14,23 +26,13 @@ import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonNumber;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonString;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import javax.json.stream.JsonParsingException;
import org.apache.commons.jcs3.access.CacheAccess;
import org.apache.commons.jcs3.engine.behavior.IElementAttributes;
import org.openstreetmap.josm.data.cache.JCSCacheManager;
import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
import org.openstreetmap.josm.data.preferences.LongProperty;
@ -42,15 +44,12 @@ import org.openstreetmap.josm.spi.preferences.Config;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Logging;
import org.apache.commons.jcs3.access.CacheAccess;
import org.apache.commons.jcs3.engine.behavior.IElementAttributes;
/**
* Take a {@link MapWithAIInfo.MapWithAIType#ESRI} layer and convert it to a
* list of "true" layers.
*/
public class ESRISourceReader {
private static int INITIAL_SEARCH = 100;
private static final int INITIAL_SEARCH = 100;
/** The cache storing ESRI source information (json) */
public static final CacheAccess<String, String> SOURCE_CACHE = JCSCacheManager.getCache("mapwithai:esrisources", 5,
50_000, new File(Config.getDirs().getCacheDirectory(true), "mapwithai").getPath());
@ -89,7 +88,7 @@ public class ESRISourceReader {
* @throws IOException if any I/O error occurs
*/
public List<MapWithAIInfo> parse() throws IOException {
Pattern startReplace = Pattern.compile("\\{start\\}");
Pattern startReplace = Pattern.compile("\\{start}");
String search = "/search" + JSON_QUERY_PARAM + "&sortField=added&sortOrder=desc&num=" + INITIAL_SEARCH
+ "&start={start}";
String url = source.getUrl();

75
src/main/java/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReader.java
Wyświetl plik

@ -3,14 +3,17 @@ package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import static org.openstreetmap.josm.tools.I18n.tr;
import javax.json.JsonArray;
import javax.json.JsonObject;
import javax.json.JsonString;
import javax.json.JsonValue;
import java.awt.geom.AffineTransform;
import java.awt.geom.PathIterator;
import java.awt.geom.Rectangle2D;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -18,29 +21,18 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonObject;
import javax.json.JsonReader;
import javax.json.JsonString;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import org.openstreetmap.josm.data.coor.LatLon;
import org.openstreetmap.josm.data.imagery.ImageryInfo;
import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
import org.openstreetmap.josm.data.imagery.Shape;
import org.openstreetmap.josm.data.osm.BBox;
import org.openstreetmap.josm.io.CachedFile;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAICategory;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIType;
import org.openstreetmap.josm.tools.DefaultGeoProperty;
import org.openstreetmap.josm.tools.GeoPropertyIndex;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Logging;
import org.openstreetmap.josm.tools.Territories;
import org.openstreetmap.josm.tools.Utils;
/**
* Reader to parse the list of available MapWithAI servers from an JSON
@ -50,13 +42,8 @@ import org.openstreetmap.josm.tools.Utils;
* "https://gitlab.com/gokaart/JOSM_MapWithAI/-/blob/pages/public/json/sources.json">MapWithAI
* source</a>.
*/
public class MapWithAISourceReader implements Closeable {
public class MapWithAISourceReader extends CommonSourceReader<List<MapWithAIInfo>> implements Closeable {
private final String source;
private CachedFile cachedFile;
private boolean fastFail;
private static final int MIN_NODE_FOR_CLOSED_WAY = 2;
private static final int COORD_ARRAY_SIZE = 6;
/**
@ -77,7 +64,7 @@ public class MapWithAISourceReader implements Closeable {
* </ul>
*/
public MapWithAISourceReader(String source) {
this.source = source;
super(source);
}
/**
@ -86,31 +73,11 @@ public class MapWithAISourceReader implements Closeable {
* @param jsonObject The json of the data sources
* @return The parsed entries
*/
public static List<MapWithAIInfo> parseJson(JsonObject jsonObject) {
@Override
public List<MapWithAIInfo> parseJson(JsonObject jsonObject) {
return jsonObject.entrySet().stream().map(MapWithAISourceReader::parse).collect(Collectors.toList());
}
/**
* Parses MapWithAI source.
*
* @return list of source info
* @throws IOException if any I/O error occurs
*/
public List<MapWithAIInfo> parse() throws IOException {
List<MapWithAIInfo> entries = Collections.emptyList();
cachedFile = new CachedFile(source);
cachedFile.setFastFail(fastFail);
try (JsonReader reader = Json.createReader(cachedFile.setMaxAge(CachedFile.DAYS)
.setCachingStrategy(CachedFile.CachingStrategy.IfModifiedSince).getContentReader())) {
JsonStructure struct = reader.read();
if (JsonValue.ValueType.OBJECT == struct.getValueType()) {
JsonObject jsonObject = struct.asJsonObject();
entries = parseJson(jsonObject);
}
return entries;
}
}
private static MapWithAIInfo parse(Map.Entry<String, JsonValue> entry) {
String name = entry.getKey();
if (JsonValue.ValueType.OBJECT == entry.getValue().getValueType()) {
@ -181,7 +148,7 @@ public class MapWithAISourceReader implements Closeable {
DefaultGeoProperty prop = (DefaultGeoProperty) geoPropertyIndex.getGeoProperty();
Rectangle2D areaBounds = prop.getArea().getBounds2D();
ImageryBounds tmp = new ImageryBounds(bboxToBoundsString(new BBox(areaBounds.getMinX(),
areaBounds.getMinY(), areaBounds.getMaxX(), areaBounds.getMaxY()), ","), ",");
areaBounds.getMinY(), areaBounds.getMaxX(), areaBounds.getMaxY())), ",");
areaToShapes(prop.getArea()).forEach(tmp::addShape);
bounds.add(tmp);
}
@ -209,7 +176,7 @@ public class MapWithAISourceReader implements Closeable {
moveTo = coords;
}
defaultShape.addPoint(Float.toString(coords[1]), Float.toString(coords[0]));
} else if (type == PathIterator.SEG_CLOSE && moveTo != null && moveTo.length >= MIN_NODE_FOR_CLOSED_WAY) {
} else if (type == PathIterator.SEG_CLOSE && moveTo != null) {
defaultShape.addPoint(Float.toString(moveTo[1]), Float.toString(moveTo[0]));
shapes.add(defaultShape);
defaultShape = new Shape();
@ -224,25 +191,9 @@ public class MapWithAISourceReader implements Closeable {
return shapes;
}
private static String bboxToBoundsString(BBox bbox, String separator) {
return String.join(separator, LatLon.cDdFormatter.format(bbox.getBottomRightLat()),
private static String bboxToBoundsString(BBox bbox) {
return String.join(",", LatLon.cDdFormatter.format(bbox.getBottomRightLat()),
LatLon.cDdFormatter.format(bbox.getTopLeftLon()), LatLon.cDdFormatter.format(bbox.getTopLeftLat()),
LatLon.cDdFormatter.format(bbox.getBottomRightLon()));
}
/**
* Sets whether opening HTTP connections should fail fast, i.e., whether a
* {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
*
* @param fastFail whether opening HTTP connections should fail fast
* @see CachedFile#setFastFail(boolean)
*/
public void setFastFail(boolean fastFail) {
this.fastFail = fastFail;
}
@Override
public void close() throws IOException {
Utils.close(cachedFile);
}
}

8
test/integration/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReaderTestIT.java
Wyświetl plik

@ -7,9 +7,9 @@ import static org.openstreetmap.josm.tools.I18n.tr;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.openstreetmap.josm.plugins.mapwithai.backend.DataAvailability;
@ -19,6 +19,8 @@ import org.openstreetmap.josm.plugins.mapwithai.testutils.annotations.NoExceptio
import org.openstreetmap.josm.testutils.JOSMTestRules;
import org.openstreetmap.josm.testutils.annotations.BasicPreferences;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
/**
* Integration test for {@link MapWithAISourceReader}
*
@ -35,10 +37,10 @@ class MapWithAISourceReaderTestIT {
void testDefaultSourceIT() throws IOException {
DataAvailability.setReleaseUrl(DataAvailability.DEFAULT_SERVER_URL);
try (MapWithAISourceReader source = new MapWithAISourceReader(DataAvailability.getReleaseUrl())) {
List<MapWithAIInfo> infoList = source.parse();
List<MapWithAIInfo> infoList = source.parse().orElse(Collections.emptyList());
assertFalse(infoList.isEmpty(), "There should be viable sources");
for (MapWithAIType type : Arrays.asList(MapWithAIType.FACEBOOK, MapWithAIType.THIRD_PARTY)) {
assertTrue(infoList.stream().filter(i -> type.equals(i.getSourceType())).count() > 0,
assertTrue(infoList.stream().anyMatch(i -> type.equals(i.getSourceType())),
tr("Type {0} should have more than 0 sources", type.getTypeString()));
}
}

12
test/unit/org/openstreetmap/josm/plugins/mapwithai/io/mapwithai/MapWithAISourceReaderTest.java
Wyświetl plik

@ -4,23 +4,23 @@ package org.openstreetmap.josm.plugins.mapwithai.io.mapwithai;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.util.Collections;
import java.util.List;
import javax.json.Json;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObjectBuilder;
import javax.json.JsonValue;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.openstreetmap.josm.plugins.mapwithai.data.mapwithai.MapWithAIInfo;
import org.openstreetmap.josm.testutils.JOSMTestRules;
import org.openstreetmap.josm.testutils.annotations.BasicPreferences;
import com.google.common.collect.ImmutableMap;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.openstreetmap.josm.testutils.annotations.BasicPreferences;
@BasicPreferences
class MapWithAISourceReaderTest {
@ -32,7 +32,7 @@ class MapWithAISourceReaderTest {
void testParseSimple() {
JsonObjectBuilder builder = Json.createObjectBuilder();
builder.add("nowhere", JsonValue.NULL);
List<MapWithAIInfo> infoList = MapWithAISourceReader.parseJson(builder.build());
List<MapWithAIInfo> infoList = new MapWithAISourceReader("").parseJson(builder.build());
assertEquals(1, infoList.size());
assertEquals("nowhere", infoList.get(0).getName());
}
@ -47,7 +47,7 @@ class MapWithAISourceReaderTest {
coCountries.add("US-CO", coCountriesArray.build());
co.add("countries", coCountries.build());
builder.add("Colorado", co);
List<MapWithAIInfo> infoList = MapWithAISourceReader.parseJson(builder.build());
List<MapWithAIInfo> infoList = new MapWithAISourceReader("").parseJson(builder.build());
assertEquals(1, infoList.size());
MapWithAIInfo info = infoList.stream().filter(i -> "Colorado".equals(i.getName())).findFirst().orElse(null);
assertNotNull(info);