diff --git a/quickstart.sh b/quickstart.sh index 38bf978a..a7566996 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -19,4 +19,4 @@ fi echo "Running..." java -Dinput="./data/sources/${AREA}.pbf" \ -cp "$JAR" \ - com.onthegomap.flatmap.profiles.OpenMapTilesProfile + com.onthegomap.flatmap.OpenMapTilesMain diff --git a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java index 64f706b3..92712b5a 100644 --- a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java +++ b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java @@ -49,7 +49,7 @@ public class OpenMapTilesMain { LOGGER.info("Building OpenMapTiles profile into " + output + " in these phases:"); if (fetchWikidata) { - LOGGER.info("- [wikidata] Fetch OpenStreetMap element name translations from wikidata"); + LOGGER.info(" [wikidata] Fetch OpenStreetMap element name translations from wikidata"); } LOGGER.info(" [lake_centerlines] Extract lake centerlines"); LOGGER.info(" [water_polygons] Process ocean polygons"); @@ -78,12 +78,9 @@ public class OpenMapTilesMain { } stats.time("lake_centerlines", () -> - new ShapefileReader("EPSG:3857", centerlines, stats) - .process("lake_centerlines", renderer, featureMap, config)); + ShapefileReader.process("EPSG:3857", "lake_centerlines", centerlines, renderer, featureMap, config)); stats.time("water_polygons", () -> - new ShapefileReader(waterPolygons, stats) - .process("water_polygons", renderer, featureMap, config) - ); + ShapefileReader.process("water_polygons", waterPolygons, renderer, featureMap, config)); stats.time("natural_earth", () -> new NaturalEarthReader(naturalEarth, tmpDir.resolve("natearth.sqlite").toFile(), stats) .process("natural_earth", renderer, featureMap, config) diff --git a/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java b/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java index 7c144d24..9b359809 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java @@ -17,7 +17,12 @@ public class NaturalEarthReader extends Reader { } @Override - public SourceStep open() { + public SourceStep read() { return null; } + + @Override + public void close() { + + } } diff --git a/src/main/java/com/onthegomap/flatmap/reader/Reader.java b/src/main/java/com/onthegomap/flatmap/reader/Reader.java index e473ccd1..0771f027 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/Reader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/Reader.java @@ -12,14 +12,15 @@ import com.onthegomap.flatmap.monitoring.ProgressLoggers; import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology; import com.onthegomap.flatmap.worker.Topology.SourceStep; +import java.io.Closeable; import java.util.concurrent.atomic.AtomicLong; import org.locationtech.jts.geom.Envelope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class Reader { +public abstract class Reader implements Closeable { - private final Stats stats; + protected final Stats stats; private final Logger LOGGER = LoggerFactory.getLogger(getClass()); public Reader(Stats stats) { @@ -35,7 +36,7 @@ public abstract class Reader { AtomicLong featuresWritten = new AtomicLong(0); var topology = Topology.start(name, stats) - .fromGenerator("read", open()) + .fromGenerator("read", read()) .addBuffer("read_queue", 1000) .addWorker("process", threads, (prev, next) -> { RenderableFeatures features = new RenderableFeatures(); @@ -69,6 +70,8 @@ public abstract class Reader { public abstract long getCount(); - public abstract SourceStep open(); + public abstract SourceStep read(); + @Override + public abstract void close(); } diff --git a/src/main/java/com/onthegomap/flatmap/reader/ReaderFeature.java b/src/main/java/com/onthegomap/flatmap/reader/ReaderFeature.java new file mode 100644 index 00000000..0a5b4b0a --- /dev/null +++ b/src/main/java/com/onthegomap/flatmap/reader/ReaderFeature.java @@ -0,0 +1,18 @@ +package com.onthegomap.flatmap.reader; + +import com.onthegomap.flatmap.SourceFeature; +import org.locationtech.jts.geom.Geometry; + +public class ReaderFeature implements SourceFeature { + + private final Geometry geometry; + + public ReaderFeature(Geometry geometry) { + this.geometry = geometry; + } + + @Override + public Geometry getGeometry() { + return geometry; + } +} diff --git a/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java b/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java index 9ecf596c..b31fb996 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java @@ -1,14 +1,104 @@ package com.onthegomap.flatmap.reader; +import com.onthegomap.flatmap.FeatureRenderer; +import com.onthegomap.flatmap.FlatMapConfig; import com.onthegomap.flatmap.SourceFeature; +import com.onthegomap.flatmap.collections.MergeSortFeatureMap; import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology.SourceStep; +import java.io.Closeable; import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.geotools.data.FeatureSource; +import org.geotools.data.shapefile.ShapefileDataStore; +import org.geotools.feature.FeatureCollection; +import org.geotools.feature.FeatureIterator; +import org.geotools.geometry.jts.JTS; +import org.geotools.referencing.CRS; +import org.locationtech.jts.geom.Geometry; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.filter.Filter; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.opengis.referencing.operation.MathTransform; -public class ShapefileReader extends Reader { +public class ShapefileReader extends Reader implements Closeable { + + private final FeatureCollection inputSource; + final FeatureIterator featureIterator; + private String[] attributeNames; + private final ShapefileDataStore dataStore; + private MathTransform transform; + + public static void process(String sourceProjection, String name, File input, FeatureRenderer renderer, + MergeSortFeatureMap writer, FlatMapConfig config) { + try (var reader = new ShapefileReader(sourceProjection, input, config.stats())) { + reader.process(name, renderer, writer, config); + } + } + + public static void process(String name, File input, FeatureRenderer renderer, + MergeSortFeatureMap writer, FlatMapConfig config) { + process(null, name, input, renderer, writer, config); + } public ShapefileReader(String sourceProjection, File input, Stats stats) { super(stats); + dataStore = decode(input); + try { + String typeName = dataStore.getTypeNames()[0]; + FeatureSource source = + dataStore.getFeatureSource(typeName); + + inputSource = source.getFeatures(Filter.INCLUDE); + CoordinateReferenceSystem src = + sourceProjection == null ? source.getSchema().getCoordinateReferenceSystem() : CRS.decode(sourceProjection); + CoordinateReferenceSystem dest = CRS.decode("EPSG:4326", true); + transform = CRS.findMathTransform(src, dest); + if (transform.isIdentity()) { + transform = null; + } + attributeNames = new String[inputSource.getSchema().getAttributeCount()]; + for (int i = 0; i < attributeNames.length; i++) { + attributeNames[i] = inputSource.getSchema().getDescriptor(i).getLocalName(); + } + this.featureIterator = inputSource.features(); + } catch (IOException | FactoryException e) { + throw new RuntimeException(e); + } + } + + private ShapefileDataStore decode(File file) { + try { + final String name = file.getName(); + + URI uri; + + if (name.endsWith(".zip")) { + String shapeFileInZip; + try (ZipFile zip = new ZipFile(file)) { + shapeFileInZip = zip.stream() + .map(ZipEntry::getName) + .filter(z -> z.endsWith(".shp")) + .findAny().orElse(null); + } + if (shapeFileInZip == null) { + throw new IllegalArgumentException("No .shp file found inside " + name); + } + uri = URI.create("jar:file:" + file.toPath().toAbsolutePath() + "!/" + shapeFileInZip); + } else if (name.endsWith(".shp")) { + uri = file.toURI(); + } else { + throw new IllegalArgumentException("Invalid shapefile input: " + file + " must be zip or shp"); + } + return new ShapefileDataStore(uri.toURL()); + } catch (IOException e) { + throw new RuntimeException(e); + } } public ShapefileReader(File input, Stats stats) { @@ -17,11 +107,34 @@ public class ShapefileReader extends Reader { @Override public long getCount() { - return 0; + return inputSource.size(); } @Override - public SourceStep open() { - return null; + public SourceStep read() { + return next -> { + while (featureIterator.hasNext()) { + SimpleFeature feature = featureIterator.next(); + Geometry source = (Geometry) feature.getDefaultGeometry(); + Geometry transformed = source; + if (transform != null) { + transformed = JTS.transform(source, transform); + } + if (transformed != null) { + SourceFeature geom = new ReaderFeature(transformed); + // TODO + // for (int i = 1; i < attributeNames.length; i++) { + // geom.setTag(attributeNames[i], feature.getAttribute(i)); + // } + next.accept(geom); + } + } + }; + } + + @Override + public void close() { + featureIterator.close(); + dataStore.dispose(); } } diff --git a/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java b/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java index aad7f7cf..001779e3 100644 --- a/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java +++ b/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java @@ -13,16 +13,16 @@ import org.junit.jupiter.api.Timeout; public class OsmInputFileTest { - private OsmInputFile file = new OsmInputFile(new File("src/test/resources/andorra-latest.osm.pbf")); + private OsmInputFile file = new OsmInputFile(new File("src/test/resources/monaco-latest.osm.pbf")); @Test public void testGetBounds() { - assertArrayEquals(new double[]{1.412368, 42.4276, 1.787481, 42.65717}, file.getBounds()); + assertArrayEquals(new double[]{7.409205, 43.72335, 7.448637, 43.75169}, file.getBounds()); } @Test @Timeout(30) - public void testReadAndorraTwice() { + public void testReadMonacoTwice() { for (int i = 1; i <= 2; i++) { AtomicInteger nodes = new AtomicInteger(0); AtomicInteger ways = new AtomicInteger(0); @@ -37,9 +37,9 @@ public class OsmInputFileTest { case ReaderElement.RELATION -> rels.incrementAndGet(); } }).await(); - assertEquals(246_028, nodes.get(), "nodes pass " + i); - assertEquals(12_677, ways.get(), "ways pass " + i); - assertEquals(287, rels.get(), "rels pass " + i); + assertEquals(25_423, nodes.get(), "nodes pass " + i); + assertEquals(4_106, ways.get(), "ways pass " + i); + assertEquals(243, rels.get(), "rels pass " + i); } } } diff --git a/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java b/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java new file mode 100644 index 00000000..d0cdd0e3 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java @@ -0,0 +1,51 @@ +package com.onthegomap.flatmap.reader; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.onthegomap.flatmap.GeoUtils; +import com.onthegomap.flatmap.monitoring.Stats.InMemory; +import com.onthegomap.flatmap.worker.Topology; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.locationtech.jts.geom.Geometry; + +public class ShapefileReaderTest { + + private ShapefileReader reader = new ShapefileReader(new File("src/test/resources/shapefile.zip"), new InMemory()); + + @AfterEach + public void close() { + reader.close(); + } + + @Test + public void testCount() { + assertEquals(86, reader.getCount()); + } + + @Test + @Timeout(30) + public void testReadShapefile() { + Map counts = new TreeMap<>(); + List points = new ArrayList<>(); + Topology.start("test", new InMemory()) + .fromGenerator("shapefile", reader.read()) + .addBuffer("reader_queue", 100, 1) + .sinkToConsumer("counter", 1, elem -> { + String type = elem.getGeometry().getGeometryType(); + counts.put(type, counts.getOrDefault(type, 0) + 1); + points.add(elem.getGeometry()); + }).await(); + assertEquals(86, points.size()); + var gc = GeoUtils.gf.createGeometryCollection(points.toArray(new Geometry[0])); + var centroid = gc.getCentroid(); + assertEquals(-77.0297995, centroid.getX(), 5); + assertEquals(38.9119684, centroid.getY(), 5); + } +} diff --git a/src/test/resources/andorra-latest.osm.pbf b/src/test/resources/andorra-latest.osm.pbf deleted file mode 100644 index 8a789c3d..00000000 Binary files a/src/test/resources/andorra-latest.osm.pbf and /dev/null differ diff --git a/src/test/resources/monaco-latest.osm.pbf b/src/test/resources/monaco-latest.osm.pbf new file mode 100644 index 00000000..36ee3550 Binary files /dev/null and b/src/test/resources/monaco-latest.osm.pbf differ diff --git a/src/test/resources/shapefile.zip b/src/test/resources/shapefile.zip new file mode 100644 index 00000000..84884ee8 Binary files /dev/null and b/src/test/resources/shapefile.zip differ