From 15b939b2b717d804e7d2fb50f0be8d25e0d466b9 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Thu, 3 Aug 2023 21:26:07 -0400 Subject: [PATCH] fixes --- .../onthegomap/planetiler/config/Bounds.java | 4 ++ .../planetiler/overture/Overture.java | 33 ++++++++--- .../planetiler/overture/OvertureSchema.java | 15 +++++ .../reader/parquet/AvroParquetReader.java | 55 ++++++++++++------- .../reader/parquet/ParquetInputFile.java | 7 ++- .../java/com/onthegomap/planetiler/Main.java | 4 +- 6 files changed, 86 insertions(+), 32 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java index f77560bf..a6243a8c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java @@ -28,6 +28,10 @@ public class Bounds { set(latLon); } + public boolean isWorld() { + return latLon == null || latLon.contains(GeoUtils.WORLD_LAT_LON_BOUNDS); + } + public Envelope latLon() { return latLon == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : latLon; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java index c67aff99..647c02e2 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java @@ -27,12 +27,10 @@ public class Overture implements Profile { private final boolean connectors; private final boolean metadata; + private final PlanetilerConfig config; Overture(PlanetilerConfig config) { - // 650mb no connectors, no metadata - // 1gb connectors, no metadata - // 1.7gb connectors, metadata - // 932mb no connectors, metadata + this.config = config; this.connectors = config.arguments().getBoolean("connectors", "include connectors", false); this.metadata = config.arguments().getBoolean("metadata", "include element metadata (version, update time, id)", false); @@ -41,14 +39,17 @@ public class Overture implements Profile { public static void main(String[] args) throws Exception { var base = Path.of("data", "sources", "overture"); var arguments = Arguments.fromEnvOrArgs(args); - var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", true); + var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", false); + var release = arguments.getString("release", "overture release", "2023-07-26-alpha.0"); var pt = Planetiler.create(arguments) .addAvroParquetSource("overture", base) .setProfile(planetiler -> new Overture(planetiler.config())) .overwriteOutput(Path.of("data", "output.pmtiles")); - downloadFiles(base, pt, "2023-07-26-alpha.0", sample); + if (arguments.getBoolean("download", "download overture files", true)) { + downloadFiles(base, pt, release, sample); + } pt.run(); } @@ -91,10 +92,11 @@ public class Overture implements Profile { @Override public List postProcessLayerFeatures(String layer, int zoom, List items) throws GeometryException { + double tolerance = config.tolerance(zoom); if (layer.equals("admins/administrativeBoundary")) { - return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true); + return FeatureMerge.mergeLineStrings(items, 0, tolerance, 4, true); } else if (layer.equals("transportation/segment")) { - return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true); + return FeatureMerge.mergeLineStrings(items, 0.25, tolerance, 4, true); } return items; } @@ -104,6 +106,18 @@ public class Overture implements Profile { return "Overture"; } + @Override + public String attribution() { + return """ + © OpenStreetMap contributors + USGS + © Microsoft + © Esri Community Maps contributors + """ + .replaceAll("\n", " ") + .trim(); + } + private void processOvertureFeature(OvertureSchema.Segment element, SourceFeature sourceFeature, FeatureCollector features) { int minzoom = switch (element.subType()) { @@ -129,6 +143,9 @@ public class Overture implements Profile { case RAIL -> 8; case WATER -> 10; }; + if (element.road() != null && element.road().flags() != null && element.road().flags().contains("isLink")) { + minzoom = Math.max(minzoom, 9); + } var feature = features.line(sourceFeature.getSourceLayer()) .setMinZoom(minzoom) .setAttr("width", element.width()) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/OvertureSchema.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/OvertureSchema.java index b3146ebe..89a34427 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/OvertureSchema.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/OvertureSchema.java @@ -101,6 +101,11 @@ public class OvertureSchema { public static BuildingClass parse(Object obj) { return obj == null ? null : BuildingClass.valueOf(obj.toString().toUpperCase()); } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } } public enum DrivingSide { @@ -110,6 +115,11 @@ public class OvertureSchema { public static DrivingSide parse(Object str) { return str == null ? null : DrivingSide.valueOf(str.toString().toUpperCase()); } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } } public record Name(String value, String language) { @@ -260,6 +270,11 @@ public class OvertureSchema { public static SegmentSubType parse(Object obj) { return obj == null ? null : SegmentSubType.valueOf(obj.toString().toUpperCase()); } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } } record Partial (List value, Range at) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/AvroParquetReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/AvroParquetReader.java index 33e06589..5d932752 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/AvroParquetReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/AvroParquetReader.java @@ -22,6 +22,8 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.ToLongFunction; import org.apache.avro.generic.GenericRecord; +import org.apache.parquet.filter2.compat.FilterCompat; +import org.apache.parquet.filter2.predicate.FilterApi; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; @@ -86,14 +88,23 @@ public class AvroParquetReader { public void process(List sourcePath, FeatureGroup writer, PlanetilerConfig config) { var timer = stats.startStage(sourceName); - var inputFiles = sourcePath.stream().map(ParquetInputFile::new).toList(); + Envelope latLonBounds = config.bounds().latLon(); + FilterCompat.Filter filter = config.bounds().isWorld() ? FilterCompat.NOOP : FilterCompat.get(FilterApi.and( + FilterApi.and( + FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxx"), latLonBounds.getMinX()), + FilterApi.ltEq(FilterApi.doubleColumn("bbox.minx"), latLonBounds.getMaxX()) + ), + FilterApi.and( + FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxy"), latLonBounds.getMinY()), + FilterApi.ltEq(FilterApi.doubleColumn("bbox.miny"), latLonBounds.getMaxY()) + ) + )); + var inputFiles = sourcePath.stream().map(path -> new ParquetInputFile(path, filter)).toList(); long featureCount = inputFiles.stream().mapToLong(ParquetInputFile::getCount).sum(); long blockCount = inputFiles.stream().mapToLong(ParquetInputFile::getBlockCount).sum(); int readThreads = config.featureReadThreads(); int processThreads = config.featureProcessThreads(); int writeThreads = config.featureWriteThreads(); - // TODO push down bounds predicate to reader - Envelope latLonBounds = config.bounds().latLon(); var blocksRead = Counter.newMultiThreadCounter(); var featuresRead = Counter.newMultiThreadCounter(); var featuresWritten = Counter.newMultiThreadCounter(); @@ -108,7 +119,7 @@ public class AvroParquetReader { } } }) - .addBuffer("row_groups", Math.max(10, processThreads / 2)) + .addBuffer("row_groups", 10) .addWorker("process", processThreads, (prev, next) -> { var blocks = blocksRead.counterForThread(); var elements = featuresRead.counterForThread(); @@ -116,24 +127,26 @@ public class AvroParquetReader { try (FeatureRenderer renderer = newFeatureRenderer(writer, config, next)) { for (var block : prev) { for (var item : block) { - var sourceFeature = new AvroParquetFeature( - item, - sourceName, - layerParser.apply(block.getFileName()), - block.getFileName(), - idParser.applyAsLong(item), - geometryParser, - toMap(item) - ); - FeatureCollector features = featureCollectors.get(sourceFeature); - try { - profile.processFeature(sourceFeature, features); - for (FeatureCollector.Feature renderable : features) { - renderer.accept(renderable); + if (item != null) { + var sourceFeature = new AvroParquetFeature( + item, + sourceName, + layerParser.apply(block.getFileName()), + block.getFileName(), + idParser.applyAsLong(item), + geometryParser, + toMap(item) + ); + FeatureCollector features = featureCollectors.get(sourceFeature); + try { + profile.processFeature(sourceFeature, features); + for (FeatureCollector.Feature renderable : features) { + renderer.accept(renderable); + } + } catch (Exception e) { + e.printStackTrace(); + LOGGER.error("Error processing {}", sourceFeature, e); } - } catch (Exception e) { - e.printStackTrace(); - LOGGER.error("Error processing {}", sourceFeature, e); } elements.inc(); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java index 9216f10b..a5b996cf 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java @@ -31,9 +31,11 @@ public class ParquetInputFile { private final ParquetMetadata metadata; private final InputFile inputFile; private final Path path; + private final FilterCompat.Filter filter; - public ParquetInputFile(Path path) { + public ParquetInputFile(Path path, FilterCompat.Filter filter) { this.path = path; + this.filter = filter; inputFile = new InputFile() { @Override public long getLength() { @@ -99,6 +101,7 @@ public class ParquetInputFile { return metadata.getBlocks().stream().map(block -> { try { // happens in reader thread + // TODO read smaller set of rows to reduce memory usage var group = reader.readRowGroup(block.getOrdinal()); return (Block) new Block() { @Override @@ -113,7 +116,7 @@ public class ParquetInputFile { var readContext = readSupport.init(new InitContext(config, keyValueMetadataSets, schema)); MessageColumnIO columnIO = columnIOFactory.getColumnIO(schema); var converter = readSupport.prepareForRead(config, keyValueMetadata, schema, readContext); - var recordReader = columnIO.getRecordReader(group, converter, FilterCompat.NOOP); + var recordReader = columnIO.getRecordReader(group, converter, filter); long total = block.getRowCount(); return new Iterator<>() { long i = 0; diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index dd13ec05..de07fa5d 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -11,6 +11,7 @@ import com.onthegomap.planetiler.examples.OsmQaTiles; import com.onthegomap.planetiler.examples.ToiletsOverlay; import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi; import com.onthegomap.planetiler.mbtiles.Verify; +import com.onthegomap.planetiler.overture.Overture; import java.util.Arrays; import java.util.Locale; import java.util.Map; @@ -50,7 +51,8 @@ public class Main { entry("benchmark-longlongmap", LongLongMapBench::main), entry("verify-mbtiles", Verify::main), - entry("verify-monaco", VerifyMonaco::main) + entry("verify-monaco", VerifyMonaco::main), + entry("overture", Overture::main) ); private static EntryPoint bundledSchema(String path) {