kopia lustrzana https://github.com/onthegomap/planetiler
Small benchmarks (#801)
Michael Barry
GitHub
rodzic
08990007a9
commit
328e1b4d53
12
README.md
12
README.md
|
|
@ -172,11 +172,13 @@ Some example runtimes for the OpenMapTiles profile (excluding downloading resour
|
|||
|
||||
Merging nearby buildings at z13 is very expensive, when run with `--building-merge-z13=false`:
|
||||
|
||||
| Input | Version | Machine | Time | mbtiles size | Logs |
|
||||
|------------------------------------------------|---------|---------------------------------|-------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| s3://osm-pds/2022/planet-220530.osm.pbf (69GB) | 0.5.0 | c2d-standard-112 (112cpu/448GB) | 26m cpu:27h47m avg:63.9 | 79GB | [logs](planet-logs/v0.5.0-planet-c2d-standard-112-no-z13-building-merge.txt) |
|
||||
| s3://osm-pds/2024/planet-240108.osm.pbf (73GB) | 0.7.0 | c7gd.16xlarge (64cpu/128GB) | 29m cpu:23h57 avg:50 | 69GB pmtiles | [logs](planet-logs/v0.7.0-planet-c7gd-128gb.txt) |
|
||||
| s3://osm-pds/2022/planet-220530.osm.pbf (69GB) | 0.5.0 | c6gd.16xlarge (64cpu/128GB) | 39m cpu:27h4m avg:42.1 | 79GB | [logs](planet-logs/v0.5.0-planet-c6gd-128gb-no-z13-building-merge.txt), [VisualVM Profile](planet-logs/v0.5.0-planet-c6gd-128gb-no-z13-building-merge.nps) |
|
||||
| Input | Version | Machine | Time | mbtiles size | Logs |
|
||||
|------------------------------------------------|---------|---------------------------------|--------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| s3://osm-pds/2022/planet-220530.osm.pbf (69GB) | 0.5.0 | c2d-standard-112 (112cpu/448GB) | 26m cpu:27h47m avg:63.9 | 79GB | [logs](planet-logs/v0.5.0-planet-c2d-standard-112-no-z13-building-merge.txt) |
|
||||
| s3://osm-pds/2024/planet-240108.osm.pbf (73GB) | 0.7.0 | c7gd.16xlarge (64cpu/128GB) | 29m cpu:23h57 avg:50 | 69GB pmtiles | [logs](planet-logs/v0.7.0-planet-c7gd-128gb-no-z13-building-merge.txt) |
|
||||
| s3://osm-pds/2024/planet-240108.osm.pbf (73GB) | 0.7.0 | c7gd.2xlarge (8cpu/16GB) | 3h35m cpu:19h45 avg:5.5 | 69GB pmtiles | [logs](planet-logs/v0.7.0-planet-c7gd-16gb-no-z13-building-merge.txt) |
|
||||
| s3://osm-pds/2024/planet-240108.osm.pbf (73GB) | 0.7.0 | im4gn.large (2cpu/8GB) | 18h18m cpu:28h6m avg:1.5 | 69GB pmtiles | [logs](planet-logs/v0.7.0-planet-im4gn-8gb-no-z13-building-merge.txt) |
|
||||
| s3://osm-pds/2022/planet-220530.osm.pbf (69GB) | 0.5.0 | c6gd.16xlarge (64cpu/128GB) | 39m cpu:27h4m avg:42.1 | 79GB | [logs](planet-logs/v0.5.0-planet-c6gd-128gb-no-z13-building-merge.txt), [VisualVM Profile](planet-logs/v0.5.0-planet-c6gd-128gb-no-z13-building-merge.nps) |
|
||||
|
||||
## Alternatives
|
||||
|
||||
|
|
|
|||
Plik diff jest za duży
Load Diff
Plik diff jest za duży
Load Diff
|
|
@ -91,7 +91,7 @@ public class TileArchiveWriter {
|
|||
int chunksToRead = Math.max(1, features.chunksToRead());
|
||||
int readThreads = Math.min(config.featureReadThreads(), chunksToRead);
|
||||
int threads = config.threads();
|
||||
int processThreads = threads < 10 ? threads : threads - readThreads;
|
||||
int processThreads = threads < 8 ? threads : (threads - readThreads);
|
||||
int tileWriteThreads = config.tileWriteThreads();
|
||||
|
||||
// when using more than 1 read thread: (N read threads) -> (1 merge thread) -> ...
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public record PlanetilerConfig(
|
|||
Math.max(1, (threads - 16) / 32 + 1));
|
||||
int featureProcessThreads =
|
||||
arguments.getInteger("process_threads", "number of threads to use when processing input features",
|
||||
Math.max(threads < 4 ? threads : (threads - featureWriteThreads), 1));
|
||||
Math.max(threads < 8 ? threads : (threads - featureWriteThreads), 1));
|
||||
Bounds bounds = new Bounds(arguments.bounds("bounds", "bounds"));
|
||||
Path polygonFile =
|
||||
arguments.file("polygon", "a .poly file that limits output to tiles intersecting the shape", null);
|
||||
|
|
|
|||
|
|
@ -159,10 +159,11 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
.addProcessStats()
|
||||
.addInMemoryObject("hppc", this)
|
||||
.newLine();
|
||||
int threads = config.threads();
|
||||
|
||||
if (nodeLocationDb instanceof LongLongMap.ParallelWrites) {
|
||||
// If the node location writer supports parallel writes, then parse, process, and write node locations from worker threads
|
||||
int parseThreads = Math.max(1, config.threads() - 1);
|
||||
int parseThreads = Math.max(1, threads < 8 ? threads : (threads - 1));
|
||||
pass1Phaser.registerWorkers(parseThreads);
|
||||
var parallelPipeline = pipeline
|
||||
.fromGenerator("read", osmBlockSource::forEachBlock)
|
||||
|
|
@ -174,7 +175,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
// If the node location writer requires sequential writes, then the reader hands off the block to workers
|
||||
// and a handle that the result will go on to the single-threaded writer, and the writer emits new nodes when
|
||||
// they are ready
|
||||
int parseThreads = Math.max(1, config.threads() - 2);
|
||||
int parseThreads = Math.max(1, threads < 8 ? threads : (threads - 2));
|
||||
int pendingBlocks = parseThreads * 2;
|
||||
// Each worker will hand off finished elements to the single process thread. A Future<List<OsmElement>> would result
|
||||
// in too much memory usage/GC so use a WeightedHandoffQueue instead which will fill up with lightweight objects
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue