fixes

planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java

@@ -28,6 +28,10 @@ public class Bounds {
     set(latLon);
   }
 
+  public boolean isWorld() {
+    return latLon == null || latLon.contains(GeoUtils.WORLD_LAT_LON_BOUNDS);
+  }
+
   public Envelope latLon() {
     return latLon == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : latLon;
   }

planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java

@@ -27,12 +27,10 @@ public class Overture implements Profile {
 
   private final boolean connectors;
   private final boolean metadata;
+  private final PlanetilerConfig config;
 
   Overture(PlanetilerConfig config) {
-    // 650mb no connectors, no metadata
-    // 1gb connectors, no metadata
-    // 1.7gb connectors, metadata
-    // 932mb no connectors, metadata
+    this.config = config;
     this.connectors = config.arguments().getBoolean("connectors", "include connectors", false);
     this.metadata =
       config.arguments().getBoolean("metadata", "include element metadata (version, update time, id)", false);
@@ -41,14 +39,17 @@ public class Overture implements Profile {
   public static void main(String[] args) throws Exception {
     var base = Path.of("data", "sources", "overture");
     var arguments = Arguments.fromEnvOrArgs(args);
-    var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", true);
+    var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", false);
+    var release = arguments.getString("release", "overture release", "2023-07-26-alpha.0");
 
     var pt = Planetiler.create(arguments)
       .addAvroParquetSource("overture", base)
       .setProfile(planetiler -> new Overture(planetiler.config()))
       .overwriteOutput(Path.of("data", "output.pmtiles"));
 
-    downloadFiles(base, pt, "2023-07-26-alpha.0", sample);
+    if (arguments.getBoolean("download", "download overture files", true)) {
+      downloadFiles(base, pt, release, sample);
+    }
 
     pt.run();
   }
@@ -91,10 +92,11 @@ public class Overture implements Profile {
   @Override
   public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items)
     throws GeometryException {
+    double tolerance = config.tolerance(zoom);
     if (layer.equals("admins/administrativeBoundary")) {
-      return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true);
+      return FeatureMerge.mergeLineStrings(items, 0, tolerance, 4, true);
     } else if (layer.equals("transportation/segment")) {
-      return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true);
+      return FeatureMerge.mergeLineStrings(items, 0.25, tolerance, 4, true);
     }
     return items;
   }
@@ -104,6 +106,18 @@ public class Overture implements Profile {
     return "Overture";
   }
 
+  @Override
+  public String attribution() {
+    return """
+      <a href="https://www.openstreetmap.org/copyright" target="_blank">&copy; OpenStreetMap contributors</a>
+      <a href="https://www.usgs.gov/3d-elevation-program" target="_blank">USGS</a>
+      <a href="https://github.com/microsoft/GlobalMLBuildingFootprints" target="_blank">&copy; Microsoft</a>
+      &copy; Esri Community Maps contributors
+      """
+      .replaceAll("\n", " ")
+      .trim();
+  }
+
   private void processOvertureFeature(OvertureSchema.Segment element, SourceFeature sourceFeature,
     FeatureCollector features) {
     int minzoom = switch (element.subType()) {
@@ -129,6 +143,9 @@ public class Overture implements Profile {
       case RAIL -> 8;
       case WATER -> 10;
     };
+    if (element.road() != null && element.road().flags() != null && element.road().flags().contains("isLink")) {
+      minzoom = Math.max(minzoom, 9);
+    }
     var feature = features.line(sourceFeature.getSourceLayer())
       .setMinZoom(minzoom)
       .setAttr("width", element.width())

planetiler-core/src/main/java/com/onthegomap/planetiler/overture/OvertureSchema.java

@@ -101,6 +101,11 @@ public class OvertureSchema {
     public static BuildingClass parse(Object obj) {
       return obj == null ? null : BuildingClass.valueOf(obj.toString().toUpperCase());
     }
+
+    @Override
+    public String toString() {
+      return super.toString().toLowerCase();
+    }
   }
 
   public enum DrivingSide {
@@ -110,6 +115,11 @@ public class OvertureSchema {
     public static DrivingSide parse(Object str) {
       return str == null ? null : DrivingSide.valueOf(str.toString().toUpperCase());
     }
+
+    @Override
+    public String toString() {
+      return super.toString().toLowerCase();
+    }
   }
 
   public record Name(String value, String language) {
@@ -260,6 +270,11 @@ public class OvertureSchema {
     public static SegmentSubType parse(Object obj) {
       return obj == null ? null : SegmentSubType.valueOf(obj.toString().toUpperCase());
     }
+
+    @Override
+    public String toString() {
+      return super.toString().toLowerCase();
+    }
   }
 
   record Partial<T> (List<T> value, Range at) {

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/AvroParquetReader.java

@@ -22,6 +22,8 @@ import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.ToLongFunction;
 import org.apache.avro.generic.GenericRecord;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.predicate.FilterApi;
 import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.io.ParseException;
@@ -86,14 +88,23 @@ public class AvroParquetReader {
 
   public void process(List<Path> sourcePath, FeatureGroup writer, PlanetilerConfig config) {
     var timer = stats.startStage(sourceName);
-    var inputFiles = sourcePath.stream().map(ParquetInputFile::new).toList();
+    Envelope latLonBounds = config.bounds().latLon();
+    FilterCompat.Filter filter = config.bounds().isWorld() ? FilterCompat.NOOP : FilterCompat.get(FilterApi.and(
+      FilterApi.and(
+        FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxx"), latLonBounds.getMinX()),
+        FilterApi.ltEq(FilterApi.doubleColumn("bbox.minx"), latLonBounds.getMaxX())
+      ),
+      FilterApi.and(
+        FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxy"), latLonBounds.getMinY()),
+        FilterApi.ltEq(FilterApi.doubleColumn("bbox.miny"), latLonBounds.getMaxY())
+      )
+    ));
+    var inputFiles = sourcePath.stream().map(path -> new ParquetInputFile(path, filter)).toList();
     long featureCount = inputFiles.stream().mapToLong(ParquetInputFile::getCount).sum();
     long blockCount = inputFiles.stream().mapToLong(ParquetInputFile::getBlockCount).sum();
     int readThreads = config.featureReadThreads();
     int processThreads = config.featureProcessThreads();
     int writeThreads = config.featureWriteThreads();
-    // TODO push down bounds predicate to reader
-    Envelope latLonBounds = config.bounds().latLon();
     var blocksRead = Counter.newMultiThreadCounter();
     var featuresRead = Counter.newMultiThreadCounter();
     var featuresWritten = Counter.newMultiThreadCounter();
@@ -108,7 +119,7 @@ public class AvroParquetReader {
           }
         }
       })
-      .addBuffer("row_groups", Math.max(10, processThreads / 2))
+      .addBuffer("row_groups", 10)
       .<SortableFeature>addWorker("process", processThreads, (prev, next) -> {
         var blocks = blocksRead.counterForThread();
         var elements = featuresRead.counterForThread();
@@ -116,24 +127,26 @@ public class AvroParquetReader {
         try (FeatureRenderer renderer = newFeatureRenderer(writer, config, next)) {
           for (var block : prev) {
             for (var item : block) {
-              var sourceFeature = new AvroParquetFeature(
-                item,
-                sourceName,
-                layerParser.apply(block.getFileName()),
-                block.getFileName(),
-                idParser.applyAsLong(item),
-                geometryParser,
-                toMap(item)
-              );
-              FeatureCollector features = featureCollectors.get(sourceFeature);
-              try {
-                profile.processFeature(sourceFeature, features);
-                for (FeatureCollector.Feature renderable : features) {
-                  renderer.accept(renderable);
+              if (item != null) {
+                var sourceFeature = new AvroParquetFeature(
+                  item,
+                  sourceName,
+                  layerParser.apply(block.getFileName()),
+                  block.getFileName(),
+                  idParser.applyAsLong(item),
+                  geometryParser,
+                  toMap(item)
+                );
+                FeatureCollector features = featureCollectors.get(sourceFeature);
+                try {
+                  profile.processFeature(sourceFeature, features);
+                  for (FeatureCollector.Feature renderable : features) {
+                    renderer.accept(renderable);
+                  }
+                } catch (Exception e) {
+                  e.printStackTrace();
+                  LOGGER.error("Error processing {}", sourceFeature, e);
                 }
-              } catch (Exception e) {
-                e.printStackTrace();
-                LOGGER.error("Error processing {}", sourceFeature, e);
               }
               elements.inc();
             }

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java

@@ -31,9 +31,11 @@ public class ParquetInputFile {
   private final ParquetMetadata metadata;
   private final InputFile inputFile;
   private final Path path;
+  private final FilterCompat.Filter filter;
 
-  public ParquetInputFile(Path path) {
+  public ParquetInputFile(Path path, FilterCompat.Filter filter) {
     this.path = path;
+    this.filter = filter;
     inputFile = new InputFile() {
       @Override
       public long getLength() {
@@ -99,6 +101,7 @@ public class ParquetInputFile {
           return metadata.getBlocks().stream().map(block -> {
             try {
               // happens in reader thread
+              // TODO read smaller set of rows to reduce memory usage
               var group = reader.readRowGroup(block.getOrdinal());
               return (Block) new Block() {
                 @Override
@@ -113,7 +116,7 @@ public class ParquetInputFile {
                   var readContext = readSupport.init(new InitContext(config, keyValueMetadataSets, schema));
                   MessageColumnIO columnIO = columnIOFactory.getColumnIO(schema);
                   var converter = readSupport.prepareForRead(config, keyValueMetadata, schema, readContext);
-                  var recordReader = columnIO.getRecordReader(group, converter, FilterCompat.NOOP);
+                  var recordReader = columnIO.getRecordReader(group, converter, filter);
                   long total = block.getRowCount();
                   return new Iterator<>() {
                     long i = 0;

planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java

@@ -11,6 +11,7 @@ import com.onthegomap.planetiler.examples.OsmQaTiles;
 import com.onthegomap.planetiler.examples.ToiletsOverlay;
 import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi;
 import com.onthegomap.planetiler.mbtiles.Verify;
+import com.onthegomap.planetiler.overture.Overture;
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
@@ -50,7 +51,8 @@ public class Main {
     entry("benchmark-longlongmap", LongLongMapBench::main),
 
     entry("verify-mbtiles", Verify::main),
-    entry("verify-monaco", VerifyMonaco::main)
+    entry("verify-monaco", VerifyMonaco::main),
+    entry("overture", Overture::main)
   );
 
   private static EntryPoint bundledSchema(String path) {