kopia lustrzana https://github.com/onthegomap/planetiler
fixes
rodzic
a66e57ccf9
commit
15b939b2b7
|
@ -28,6 +28,10 @@ public class Bounds {
|
|||
set(latLon);
|
||||
}
|
||||
|
||||
public boolean isWorld() {
|
||||
return latLon == null || latLon.contains(GeoUtils.WORLD_LAT_LON_BOUNDS);
|
||||
}
|
||||
|
||||
public Envelope latLon() {
|
||||
return latLon == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : latLon;
|
||||
}
|
||||
|
|
|
@ -27,12 +27,10 @@ public class Overture implements Profile {
|
|||
|
||||
private final boolean connectors;
|
||||
private final boolean metadata;
|
||||
private final PlanetilerConfig config;
|
||||
|
||||
Overture(PlanetilerConfig config) {
|
||||
// 650mb no connectors, no metadata
|
||||
// 1gb connectors, no metadata
|
||||
// 1.7gb connectors, metadata
|
||||
// 932mb no connectors, metadata
|
||||
this.config = config;
|
||||
this.connectors = config.arguments().getBoolean("connectors", "include connectors", false);
|
||||
this.metadata =
|
||||
config.arguments().getBoolean("metadata", "include element metadata (version, update time, id)", false);
|
||||
|
@ -41,14 +39,17 @@ public class Overture implements Profile {
|
|||
public static void main(String[] args) throws Exception {
|
||||
var base = Path.of("data", "sources", "overture");
|
||||
var arguments = Arguments.fromEnvOrArgs(args);
|
||||
var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", true);
|
||||
var sample = arguments.getBoolean("sample", "only download smallest file from parquet source", false);
|
||||
var release = arguments.getString("release", "overture release", "2023-07-26-alpha.0");
|
||||
|
||||
var pt = Planetiler.create(arguments)
|
||||
.addAvroParquetSource("overture", base)
|
||||
.setProfile(planetiler -> new Overture(planetiler.config()))
|
||||
.overwriteOutput(Path.of("data", "output.pmtiles"));
|
||||
|
||||
downloadFiles(base, pt, "2023-07-26-alpha.0", sample);
|
||||
if (arguments.getBoolean("download", "download overture files", true)) {
|
||||
downloadFiles(base, pt, release, sample);
|
||||
}
|
||||
|
||||
pt.run();
|
||||
}
|
||||
|
@ -91,10 +92,11 @@ public class Overture implements Profile {
|
|||
@Override
|
||||
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items)
|
||||
throws GeometryException {
|
||||
double tolerance = config.tolerance(zoom);
|
||||
if (layer.equals("admins/administrativeBoundary")) {
|
||||
return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true);
|
||||
return FeatureMerge.mergeLineStrings(items, 0, tolerance, 4, true);
|
||||
} else if (layer.equals("transportation/segment")) {
|
||||
return FeatureMerge.mergeLineStrings(items, 0, 0.125, 4, true);
|
||||
return FeatureMerge.mergeLineStrings(items, 0.25, tolerance, 4, true);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
@ -104,6 +106,18 @@ public class Overture implements Profile {
|
|||
return "Overture";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String attribution() {
|
||||
return """
|
||||
<a href="https://www.openstreetmap.org/copyright" target="_blank">© OpenStreetMap contributors</a>
|
||||
<a href="https://www.usgs.gov/3d-elevation-program" target="_blank">USGS</a>
|
||||
<a href="https://github.com/microsoft/GlobalMLBuildingFootprints" target="_blank">© Microsoft</a>
|
||||
© Esri Community Maps contributors
|
||||
"""
|
||||
.replaceAll("\n", " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
private void processOvertureFeature(OvertureSchema.Segment element, SourceFeature sourceFeature,
|
||||
FeatureCollector features) {
|
||||
int minzoom = switch (element.subType()) {
|
||||
|
@ -129,6 +143,9 @@ public class Overture implements Profile {
|
|||
case RAIL -> 8;
|
||||
case WATER -> 10;
|
||||
};
|
||||
if (element.road() != null && element.road().flags() != null && element.road().flags().contains("isLink")) {
|
||||
minzoom = Math.max(minzoom, 9);
|
||||
}
|
||||
var feature = features.line(sourceFeature.getSourceLayer())
|
||||
.setMinZoom(minzoom)
|
||||
.setAttr("width", element.width())
|
||||
|
|
|
@ -101,6 +101,11 @@ public class OvertureSchema {
|
|||
public static BuildingClass parse(Object obj) {
|
||||
return obj == null ? null : BuildingClass.valueOf(obj.toString().toUpperCase());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return super.toString().toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
public enum DrivingSide {
|
||||
|
@ -110,6 +115,11 @@ public class OvertureSchema {
|
|||
public static DrivingSide parse(Object str) {
|
||||
return str == null ? null : DrivingSide.valueOf(str.toString().toUpperCase());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return super.toString().toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
public record Name(String value, String language) {
|
||||
|
@ -260,6 +270,11 @@ public class OvertureSchema {
|
|||
public static SegmentSubType parse(Object obj) {
|
||||
return obj == null ? null : SegmentSubType.valueOf(obj.toString().toUpperCase());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return super.toString().toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
record Partial<T> (List<T> value, Range at) {
|
||||
|
|
|
@ -22,6 +22,8 @@ import java.util.function.Consumer;
|
|||
import java.util.function.Function;
|
||||
import java.util.function.ToLongFunction;
|
||||
import org.apache.avro.generic.GenericRecord;
|
||||
import org.apache.parquet.filter2.compat.FilterCompat;
|
||||
import org.apache.parquet.filter2.predicate.FilterApi;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.io.ParseException;
|
||||
|
@ -86,14 +88,23 @@ public class AvroParquetReader {
|
|||
|
||||
public void process(List<Path> sourcePath, FeatureGroup writer, PlanetilerConfig config) {
|
||||
var timer = stats.startStage(sourceName);
|
||||
var inputFiles = sourcePath.stream().map(ParquetInputFile::new).toList();
|
||||
Envelope latLonBounds = config.bounds().latLon();
|
||||
FilterCompat.Filter filter = config.bounds().isWorld() ? FilterCompat.NOOP : FilterCompat.get(FilterApi.and(
|
||||
FilterApi.and(
|
||||
FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxx"), latLonBounds.getMinX()),
|
||||
FilterApi.ltEq(FilterApi.doubleColumn("bbox.minx"), latLonBounds.getMaxX())
|
||||
),
|
||||
FilterApi.and(
|
||||
FilterApi.gtEq(FilterApi.doubleColumn("bbox.maxy"), latLonBounds.getMinY()),
|
||||
FilterApi.ltEq(FilterApi.doubleColumn("bbox.miny"), latLonBounds.getMaxY())
|
||||
)
|
||||
));
|
||||
var inputFiles = sourcePath.stream().map(path -> new ParquetInputFile(path, filter)).toList();
|
||||
long featureCount = inputFiles.stream().mapToLong(ParquetInputFile::getCount).sum();
|
||||
long blockCount = inputFiles.stream().mapToLong(ParquetInputFile::getBlockCount).sum();
|
||||
int readThreads = config.featureReadThreads();
|
||||
int processThreads = config.featureProcessThreads();
|
||||
int writeThreads = config.featureWriteThreads();
|
||||
// TODO push down bounds predicate to reader
|
||||
Envelope latLonBounds = config.bounds().latLon();
|
||||
var blocksRead = Counter.newMultiThreadCounter();
|
||||
var featuresRead = Counter.newMultiThreadCounter();
|
||||
var featuresWritten = Counter.newMultiThreadCounter();
|
||||
|
@ -108,7 +119,7 @@ public class AvroParquetReader {
|
|||
}
|
||||
}
|
||||
})
|
||||
.addBuffer("row_groups", Math.max(10, processThreads / 2))
|
||||
.addBuffer("row_groups", 10)
|
||||
.<SortableFeature>addWorker("process", processThreads, (prev, next) -> {
|
||||
var blocks = blocksRead.counterForThread();
|
||||
var elements = featuresRead.counterForThread();
|
||||
|
@ -116,24 +127,26 @@ public class AvroParquetReader {
|
|||
try (FeatureRenderer renderer = newFeatureRenderer(writer, config, next)) {
|
||||
for (var block : prev) {
|
||||
for (var item : block) {
|
||||
var sourceFeature = new AvroParquetFeature(
|
||||
item,
|
||||
sourceName,
|
||||
layerParser.apply(block.getFileName()),
|
||||
block.getFileName(),
|
||||
idParser.applyAsLong(item),
|
||||
geometryParser,
|
||||
toMap(item)
|
||||
);
|
||||
FeatureCollector features = featureCollectors.get(sourceFeature);
|
||||
try {
|
||||
profile.processFeature(sourceFeature, features);
|
||||
for (FeatureCollector.Feature renderable : features) {
|
||||
renderer.accept(renderable);
|
||||
if (item != null) {
|
||||
var sourceFeature = new AvroParquetFeature(
|
||||
item,
|
||||
sourceName,
|
||||
layerParser.apply(block.getFileName()),
|
||||
block.getFileName(),
|
||||
idParser.applyAsLong(item),
|
||||
geometryParser,
|
||||
toMap(item)
|
||||
);
|
||||
FeatureCollector features = featureCollectors.get(sourceFeature);
|
||||
try {
|
||||
profile.processFeature(sourceFeature, features);
|
||||
for (FeatureCollector.Feature renderable : features) {
|
||||
renderer.accept(renderable);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
LOGGER.error("Error processing {}", sourceFeature, e);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
LOGGER.error("Error processing {}", sourceFeature, e);
|
||||
}
|
||||
elements.inc();
|
||||
}
|
||||
|
|
|
@ -31,9 +31,11 @@ public class ParquetInputFile {
|
|||
private final ParquetMetadata metadata;
|
||||
private final InputFile inputFile;
|
||||
private final Path path;
|
||||
private final FilterCompat.Filter filter;
|
||||
|
||||
public ParquetInputFile(Path path) {
|
||||
public ParquetInputFile(Path path, FilterCompat.Filter filter) {
|
||||
this.path = path;
|
||||
this.filter = filter;
|
||||
inputFile = new InputFile() {
|
||||
@Override
|
||||
public long getLength() {
|
||||
|
@ -99,6 +101,7 @@ public class ParquetInputFile {
|
|||
return metadata.getBlocks().stream().map(block -> {
|
||||
try {
|
||||
// happens in reader thread
|
||||
// TODO read smaller set of rows to reduce memory usage
|
||||
var group = reader.readRowGroup(block.getOrdinal());
|
||||
return (Block) new Block() {
|
||||
@Override
|
||||
|
@ -113,7 +116,7 @@ public class ParquetInputFile {
|
|||
var readContext = readSupport.init(new InitContext(config, keyValueMetadataSets, schema));
|
||||
MessageColumnIO columnIO = columnIOFactory.getColumnIO(schema);
|
||||
var converter = readSupport.prepareForRead(config, keyValueMetadata, schema, readContext);
|
||||
var recordReader = columnIO.getRecordReader(group, converter, FilterCompat.NOOP);
|
||||
var recordReader = columnIO.getRecordReader(group, converter, filter);
|
||||
long total = block.getRowCount();
|
||||
return new Iterator<>() {
|
||||
long i = 0;
|
||||
|
|
|
@ -11,6 +11,7 @@ import com.onthegomap.planetiler.examples.OsmQaTiles;
|
|||
import com.onthegomap.planetiler.examples.ToiletsOverlay;
|
||||
import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi;
|
||||
import com.onthegomap.planetiler.mbtiles.Verify;
|
||||
import com.onthegomap.planetiler.overture.Overture;
|
||||
import java.util.Arrays;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
@ -50,7 +51,8 @@ public class Main {
|
|||
entry("benchmark-longlongmap", LongLongMapBench::main),
|
||||
|
||||
entry("verify-mbtiles", Verify::main),
|
||||
entry("verify-monaco", VerifyMonaco::main)
|
||||
entry("verify-monaco", VerifyMonaco::main),
|
||||
entry("overture", Overture::main)
|
||||
);
|
||||
|
||||
private static EntryPoint bundledSchema(String path) {
|
||||
|
|
Ładowanie…
Reference in New Issue