From 9d4e7bdbf1728f5a932d69cc3e2c59bb450c89bd Mon Sep 17 00:00:00 2001 From: Taylor Smock Date: Thu, 30 Jan 2020 14:46:07 -0700 Subject: [PATCH] Initial address checking Signed-off-by: Taylor Smock --- .../plugins/mapwithai/MapWithAIPlugin.java | 3 +- .../validation/tests/StreetAddressTest.java | 216 +++++++++++++++ .../tests/StreetAddressTestTest.java | 253 ++++++++++++++++++ 3 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTest.java create mode 100644 test/unit/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTestTest.java diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/MapWithAIPlugin.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/MapWithAIPlugin.java index 7fafa30..36b52d1 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/MapWithAIPlugin.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/MapWithAIPlugin.java @@ -38,6 +38,7 @@ import org.openstreetmap.josm.plugins.mapwithai.backend.MapWithAIUploadHook; import org.openstreetmap.josm.plugins.mapwithai.backend.MergeDuplicateWaysAction; import org.openstreetmap.josm.plugins.mapwithai.data.validation.tests.ConnectingNodeInformationTest; import org.openstreetmap.josm.plugins.mapwithai.data.validation.tests.RoutingIslandsTest; +import org.openstreetmap.josm.plugins.mapwithai.data.validation.tests.StreetAddressTest; import org.openstreetmap.josm.plugins.mapwithai.data.validation.tests.StubEndsTest; import org.openstreetmap.josm.plugins.mapwithai.frontend.MapWithAIDownloadReader; import org.openstreetmap.josm.spi.preferences.Config; @@ -65,7 +66,7 @@ public final class MapWithAIPlugin extends Plugin implements Destroyable { } private final static List> VALIDATORS = Arrays.asList(RoutingIslandsTest.class, - ConnectingNodeInformationTest.class, StubEndsTest.class); + ConnectingNodeInformationTest.class, StubEndsTest.class, StreetAddressTest.class); public MapWithAIPlugin(PluginInformation info) { super(info); diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTest.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTest.java new file mode 100644 index 0000000..abb53ca --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTest.java @@ -0,0 +1,216 @@ +package org.openstreetmap.josm.plugins.mapwithai.data.validation.tests; + +import static org.openstreetmap.josm.tools.I18n.marktr; +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.openstreetmap.josm.data.osm.BBox; +import org.openstreetmap.josm.data.osm.IPrimitive; +import org.openstreetmap.josm.data.osm.IWay; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.OsmPrimitive; +import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.data.validation.Severity; +import org.openstreetmap.josm.data.validation.Test; +import org.openstreetmap.josm.data.validation.TestError; +import org.openstreetmap.josm.plugins.mapwithai.MapWithAIPlugin; +import org.openstreetmap.josm.tools.Geometry; +import org.openstreetmap.josm.tools.Pair; + +public class StreetAddressTest extends Test { + private static double BBOX_EXPANSION = 0.001; + private static String ADDR_STREET = "addr:street"; + /** + * Classified highways in order of importance + * + * Copied from {@link org.openstreetmap.josm.data.validation.tests.Highways} + */ + private static final List CLASSIFIED_HIGHWAYS = Arrays.asList("motorway", "motorway_link", "trunk", + "trunk_link", "primary", "primary_link", "secondary", "secondary_link", "tertiary", "tertiary_link", + "unclassified", "residential", "living_street"); + + public StreetAddressTest() { + super(tr("Mismatched street/street addresses ({0})", MapWithAIPlugin.NAME), + tr("Check for addr:street/street name mismatches")); + } + + @Override + public void visit(Way way) { + if (way.isUsable() && isHighway(way)) { + List addresses = getNearbyAddresses(way); + Map addressOccurance = getAddressOccurance(addresses); + createError(way, addressOccurance, addresses); + } + } + + public void createError(Way way, Map occurances, List addresses) { + String name = way.get("name"); + Collection likelyNames = getLikelyNames(occurances); + TestError.Builder error = null; + if (name == null) { + error = TestError.builder(this, Severity.WARNING, 65446500); + error.message(tr(MapWithAIPlugin.NAME), + marktr("Street with no name with {0} tags nearby, name possibly {1}"), ADDR_STREET, likelyNames) + .highlight(getAddressPOI(likelyNames, addresses)); + } else if (!likelyNames.contains(name)) { + error = TestError.builder(this, Severity.WARNING, 65446501); + error.message(tr(MapWithAIPlugin.NAME), + marktr("Street name does not match most likely name, name possibly {0}"), likelyNames) + .highlight(getAddressPOI(likelyNames, addresses)); + } + if (error != null && !likelyNames.isEmpty()) { + error.primitives(way); + errors.add(error.build()); + } + } + + /** + * Get a list of likely names from a map of occurrences + * + * @param occurances The map of Name to occurrences + * @return The string(s) with the most occurrences + */ + public static List getLikelyNames(Map occurances) { + List likelyNames = new ArrayList<>(); + Integer max = 0; + for (Entry entry : occurances.entrySet()) { + if (entry.getKey() == null || entry.getKey().trim().isEmpty()) { + continue; + } + if (entry.getValue() > max) { + max = entry.getValue(); + likelyNames.clear(); + likelyNames.add(entry.getKey()); + } else if (entry.getValue() == max) { + likelyNames.add(entry.getKey()); + } + } + return likelyNames; + } + + /** + * Get address points relevant to a set of names + * + * @param names The street names of interest + * @param addresses Potential address points + * @return POI's for the street names + */ + public static List getAddressPOI(Collection names, Collection addresses) { + return addresses.stream().filter(OsmPrimitive.class::isInstance).map(OsmPrimitive.class::cast) + .filter(p -> names.contains(p.get(ADDR_STREET))).collect(Collectors.toList()); + } + + /** + * Count the street address occurances + * + * @param addressPOI The list to count + * @return A map of street names with a count + */ + public static Map getAddressOccurance(Collection addressPOI) { + Map count = new HashMap<>(); + for (IPrimitive prim : addressPOI) { + if (prim.hasTag(ADDR_STREET)) { + int current = count.getOrDefault(prim.get(ADDR_STREET), 0); + count.put(prim.get(ADDR_STREET), ++current); + } + } + return count; + } + + /** + * Get nearby addresses to a way + * + * @param way The way to get nearby addresses from + * @return The primitives that have appropriate addr tags near to the way + */ + public static List getNearbyAddresses(Way way) { + BBox bbox = expandBBox(way.getBBox(), BBOX_EXPANSION); + List addrNodes = way.getDataSet().searchNodes(bbox).parallelStream() + .filter(StreetAddressTest::hasStreetAddressTags).collect(Collectors.toList()); + List addrWays = way.getDataSet().searchWays(bbox).parallelStream() + .filter(StreetAddressTest::hasStreetAddressTags).collect(Collectors.toList()); + List addrRelations = way.getDataSet().searchRelations(bbox).parallelStream() + .filter(StreetAddressTest::hasStreetAddressTags).collect(Collectors.toList()); + List nearbyAddresses = Stream.of(addrNodes, addrWays, addrRelations).flatMap(List::parallelStream) + .filter(prim -> StreetAddressTest.isNearestRoad(way, prim)).collect(Collectors.toList()); + + return nearbyAddresses; + } + + /** + * Check if a way is the nearest road to a primitive + * + * @param way The way to check + * @param prim The primitive to get the distance from + * @return {@code true} if the primitive is the nearest way + */ + public static boolean isNearestRoad(Way way, OsmPrimitive prim) { + BBox primBBox = expandBBox(prim.getBBox(), BBOX_EXPANSION); + List> sorted = way.getDataSet().searchWays(primBBox).parallelStream() + .filter(StreetAddressTest::isHighway).map(iway -> distanceToWay(iway, prim)) + .sorted(Comparator.comparing(p -> p.b)).collect(Collectors.toList()); + + if (!sorted.isEmpty()) { + double minDistance = sorted.get(0).b; + List nearby = sorted.stream().filter(p -> p.b - minDistance < BBOX_EXPANSION * 0.05).map(p -> p.a) + .collect(Collectors.toList()); + return nearby.contains(way); + } + return false; + } + + /** + * Get the distance to a way + * + * @param way The way to get a distance to + * @param prim The primitive to get a distance from + * @return A Pair of the distance from the primitive to the way + */ + public static Pair distanceToWay(Way way, OsmPrimitive prim) { + return new Pair<>(way, Geometry.getDistance(way, prim)); + } + + /** + * Check if the primitive has an appropriate highway tag + * + * @param prim The primitive to check + * @return {@code true} if it has a highway tag that is classified + */ + public static boolean isHighway(IPrimitive prim) { + return prim instanceof IWay && prim.hasTag("highway", CLASSIFIED_HIGHWAYS); + } + + /** + * Check if the primitive has appropriate address tags + * + * @param prim The primitive to check + * @return {@code true} if it has addr:street tags (may change) + */ + public static boolean hasStreetAddressTags(IPrimitive prim) { + return prim.hasTag(ADDR_STREET); + } + + /** + * Expand a bbox by a set amount + * + * @param bbox The bbox to expand + * @param degree The amount to expand the bbox by + * @return The bbox, for easy chaining + */ + public static BBox expandBBox(BBox bbox, double degree) { + bbox.add(bbox.getBottomRightLon() + degree, bbox.getBottomRightLat() - degree); + bbox.add(bbox.getTopLeftLon() - degree, bbox.getTopLeftLat() + degree); + return bbox; + } +} diff --git a/test/unit/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTestTest.java b/test/unit/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTestTest.java new file mode 100644 index 0000000..f0c50c5 --- /dev/null +++ b/test/unit/org/openstreetmap/josm/plugins/mapwithai/data/validation/tests/StreetAddressTestTest.java @@ -0,0 +1,253 @@ +package org.openstreetmap.josm.plugins.mapwithai.data.validation.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import org.junit.Rule; +import org.junit.Test; +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.osm.AbstractPrimitive; +import org.openstreetmap.josm.data.osm.BBox; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.IPrimitive; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.testutils.JOSMTestRules; +import org.openstreetmap.josm.tools.Geometry; +import org.openstreetmap.josm.tools.Pair; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +public class StreetAddressTestTest { + private final static String ADDR_STREET = "addr:street"; + @Rule + @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD") + public JOSMTestRules test = new JOSMTestRules().projection(); + + @Test + public void testVisitWay() throws NoSuchMethodException, SecurityException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + StreetAddressTest test = new StreetAddressTest(); + Way way1 = TestUtils.newWay("", new Node(new LatLon(0, 0)), new Node(new LatLon(1, 1))); + DataSet ds = new DataSet(); + way1.getNodes().forEach(ds::addPrimitive); + ds.addPrimitive(way1); + Node node1 = new Node(new LatLon(1, 1.00001)); + node1.put(ADDR_STREET, "Test"); + ds.addPrimitive(node1); + test.visit(way1); + assertTrue(test.getErrors().isEmpty()); + way1.put("highway", "residential"); + + test.visit(way1); + assertFalse(test.getErrors().isEmpty()); + + way1.put("name", "Test1"); + test.clear(); + test.visit(way1); + assertFalse(test.getErrors().isEmpty()); + + way1.put("name", "Test"); + test.clear(); + test.visit(way1); + assertTrue(test.getErrors().isEmpty()); + + node1.remove(ADDR_STREET); + test.clear(); + test.visit(way1); + assertTrue(test.getErrors().isEmpty()); + + way1.put("name", "Test1"); + test.clear(); + Node firstNode = way1.firstNode(); + Method setIncomplete = AbstractPrimitive.class.getDeclaredMethod("setIncomplete", boolean.class); + setIncomplete.setAccessible(true); + setIncomplete.invoke(firstNode, true); + assertTrue(way1.firstNode().isIncomplete()); + test.visit(way1); + assertTrue(test.getErrors().isEmpty()); + } + + @Test + public void testGetLikelyNames() { + Map likelyNames = new HashMap<>(); + assertTrue(StreetAddressTest.getLikelyNames(likelyNames).isEmpty()); + likelyNames.put("Test Name 1", 0); + assertEquals("Test Name 1", StreetAddressTest.getLikelyNames(likelyNames).get(0)); + likelyNames.put("Test Name 2", 1); + assertEquals("Test Name 2", StreetAddressTest.getLikelyNames(likelyNames).get(0)); + assertEquals(1, StreetAddressTest.getLikelyNames(likelyNames).size()); + likelyNames.put("Test Name 3", 1); + likelyNames.put(null, 50000); + likelyNames.put(" ", 20000); + assertEquals(2, StreetAddressTest.getLikelyNames(likelyNames).size()); + assertTrue( + StreetAddressTest.getLikelyNames(likelyNames).containsAll(Arrays.asList("Test Name 2", "Test Name 3"))); + } + + @Test + public void testGetAddressPOI() { + Node poi1 = new Node(new LatLon(0, 0)); + assertTrue(StreetAddressTest.getAddressPOI(Collections.singleton("Test Street"), Collections.singleton(poi1)) + .isEmpty()); + poi1.put(ADDR_STREET, "Test Street"); + assertEquals(poi1, StreetAddressTest + .getAddressPOI(Collections.singleton("Test Street"), Collections.singleton(poi1)).get(0)); + assertEquals(poi1, StreetAddressTest.getAddressPOI(Collections.singleton("Test Street"), + Arrays.asList(new Node(new LatLon(0, 0)), poi1, new Node(new LatLon(1, 1)))).get(0)); + } + + @Test + public void testGetAddressOccurance() { + Collection holder = new HashSet<>(); + assertTrue(StreetAddressTest.getAddressOccurance(holder).isEmpty()); + Node tNode = new Node(new LatLon(0, 0)); + holder.add(tNode); + assertTrue(StreetAddressTest.getAddressOccurance(holder).isEmpty()); + tNode.put(ADDR_STREET, "Test Road 1"); + assertEquals(1, StreetAddressTest.getAddressOccurance(holder).get("Test Road 1")); + for (int i = 0; i < 10; i++) { + Node tNode2 = new Node(tNode); + tNode2.clearOsmMetadata(); + holder.add(tNode2); + } + assertEquals(11, StreetAddressTest.getAddressOccurance(holder).get("Test Road 1")); + + tNode = new Node(tNode); + tNode.clearOsmMetadata(); + tNode.remove(ADDR_STREET); + holder.add(tNode); + assertEquals(11, StreetAddressTest.getAddressOccurance(holder).get("Test Road 1")); + assertEquals(1, StreetAddressTest.getAddressOccurance(holder).size()); + + tNode.put(ADDR_STREET, "Test Road 2"); + assertEquals(11, StreetAddressTest.getAddressOccurance(holder).get("Test Road 1")); + assertEquals(1, StreetAddressTest.getAddressOccurance(holder).get("Test Road 2")); + assertEquals(2, StreetAddressTest.getAddressOccurance(holder).size()); + } + + @Test + public void testGetNearbyAddresses() { + Way way1 = TestUtils.newWay("highway=residential", new Node(new LatLon(0, 0)), new Node(new LatLon(1, 1))); + DataSet ds = new DataSet(); + way1.getNodes().forEach(ds::addPrimitive); + ds.addPrimitive(way1); + + assertTrue(StreetAddressTest.getNearbyAddresses(way1).isEmpty()); + + Node node1 = new Node(new LatLon(1, 2)); + node1.put(ADDR_STREET, "Test1"); + ds.addPrimitive(node1); + + assertTrue(StreetAddressTest.getNearbyAddresses(way1).isEmpty()); + + Node node2 = new Node(new LatLon(1, 1.0001)); + node2.put(ADDR_STREET, "Test2"); + ds.addPrimitive(node2); + + assertEquals(1, StreetAddressTest.getNearbyAddresses(way1).size()); + assertSame(node2, StreetAddressTest.getNearbyAddresses(way1).get(0)); + + Node node3 = new Node(new LatLon(1, 0.9999)); + ds.addPrimitive(node3); + + assertSame(node2, StreetAddressTest.getNearbyAddresses(way1).get(0)); + assertEquals(1, StreetAddressTest.getNearbyAddresses(way1).size()); + + node3.put(ADDR_STREET, "Test3"); + assertTrue(StreetAddressTest.getNearbyAddresses(way1).containsAll(Arrays.asList(node2, node3))); + assertEquals(2, StreetAddressTest.getNearbyAddresses(way1).size()); + } + + @Test + public void testIsNearestRoad() { + Node node1 = new Node(new LatLon(0, 0)); + DataSet ds = new DataSet(node1); + double boxCorners = 0.0009; + Way way1 = TestUtils.newWay("", new Node(new LatLon(boxCorners, boxCorners)), + new Node(new LatLon(boxCorners, -boxCorners))); + Way way2 = TestUtils.newWay("", new Node(new LatLon(-boxCorners, boxCorners)), + new Node(new LatLon(-boxCorners, -boxCorners))); + for (Way way : Arrays.asList(way1, way2)) { + way.getNodes().forEach(ds::addPrimitive); + ds.addPrimitive(way); + } + + assertFalse(StreetAddressTest.isNearestRoad(way1, node1)); + assertFalse(StreetAddressTest.isNearestRoad(way2, node1)); + + way1.put("highway", "residential"); + way2.put("highway", "motorway"); + + assertTrue(StreetAddressTest.isNearestRoad(way1, node1)); + assertTrue(StreetAddressTest.isNearestRoad(way2, node1)); + + node1.setCoor(new LatLon(boxCorners * 0.9, boxCorners * 0.9)); + assertTrue(StreetAddressTest.isNearestRoad(way1, node1)); + assertFalse(StreetAddressTest.isNearestRoad(way2, node1)); + + node1.setCoor(new LatLon(-boxCorners * 0.9, -boxCorners * 0.9)); + assertTrue(StreetAddressTest.isNearestRoad(way2, node1)); + assertFalse(StreetAddressTest.isNearestRoad(way1, node1)); + + node1.setCoor(new LatLon(0.00005, 0.00005)); + assertFalse(StreetAddressTest.isNearestRoad(way2, node1)); + assertTrue(StreetAddressTest.isNearestRoad(way1, node1)); + } + + @Test + public void testDistanceToWay() { + Node node1 = new Node(new LatLon(0, 0)); + Way way1 = TestUtils.newWay("", new Node(new LatLon(0, 0)), new Node(new LatLon(1, 1))); + Pair distance = StreetAddressTest.distanceToWay(way1, node1); + assertSame(way1, distance.a); + assertEquals(0, distance.b, 0.0); + node1.setCoor(new LatLon(0.001, 0.001)); + + distance = StreetAddressTest.distanceToWay(way1, node1); + assertSame(way1, distance.a); + assertEquals(Geometry.getDistance(way1, node1), distance.b, 0.0); + } + + @Test + public void testIsHighway() { + Node node = new Node(new LatLon(0, 0)); + assertFalse(StreetAddressTest.isHighway(node)); + node.put(ADDR_STREET, "Test Road 1"); + assertFalse(StreetAddressTest.isHighway(node)); + + Way way = TestUtils.newWay("", node, new Node(new LatLon(1, 1))); + assertFalse(StreetAddressTest.isHighway(way)); + way.put("highway", "residential"); + assertTrue(StreetAddressTest.isHighway(way)); + } + + @Test + public void testHasStreetAddressTags() { + Node node = new Node(new LatLon(0, 0)); + assertFalse(StreetAddressTest.hasStreetAddressTags(node)); + node.put(ADDR_STREET, "Test Road 1"); + assertTrue(StreetAddressTest.hasStreetAddressTags(node)); + } + + @Test + public void testExpandBBox() { + BBox bbox = new BBox(); + bbox.add(0, 0); + assertSame(bbox, StreetAddressTest.expandBBox(bbox, 0.01)); + assertTrue(BBox.bboxesAreFunctionallyEqual(bbox, new BBox(-0.01, -0.01, 0.01, 0.01), 0.0)); + } + +}