Linode test (#1)

Add results from running test on linode.

PLANET.md

@@ -82,9 +82,9 @@ tail -f logs.txt
 ```
 
 It took 3h21m (including 12 minutes downloading source data) to generate a 99GB `output.mbtiles` file. See
-the [full logs](planet-logs/planet-logs.txt) from this run or this summary that it printed at the end. Notice that it
+the [full logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt) from this run or this summary that it printed at the end.
-spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby building polygons at
+Notice that it spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby
-z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
+building polygons at z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
 
 ```
 3:21:03 DEB [mbtiles] - Tile stats:

README.md

@@ -91,11 +91,12 @@ See the [flatmap-examples](flatmap-examples) project.
 
 Some example runtimes (excluding downloading resources):
 
-| Input | Input Size | Profile | Machine | Time | mbtiles size | Logs |
+| Input | Profile | Machine | Time | mbtiles size | Logs |
-| --- | --- | --- | --- | --- | --- | --- |
+| --- | --- | --- | --- | --- | --- |
-| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap | DO 16cpu 128GB RAM | 3h9m (cpu: 42h1m @ 13.3) | 99.6GB | [logs](planet-logs/planet-logs.txt) [VisualVM Profile](planet-logs/planet.nps) |
+| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | DO 16cpu 128GB | 3h9m cpu:42h1m avg:13.3 | 99GB | [logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-do-16cpu-128gb.nps) |
-| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries | 68.6GB | Basemap | DO 16cpu 128GB RAM | 3h13m (cpu: 43h40m @ 13.5) | 101.4GB | [logs](planet-logs/logs-daylight.txt) |
+| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries (67GB) | Basemap | DO 16cpu 128GB | 3h13m cpu:43h40m avg:13.5 | 101GB | [logs](planet-logs/v0.1.0-daylight-do-16cpu-128gb.txt) |
-| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB RAM) | 59m26s (cpu: 27h6m @ 27.4) | 97.3GB | [logs](planet-logs/planet-logs-c5ad.txt) |
+| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | Linode 50cpu 128GB | 1h9m cpu:24h36m avg:21.2 | 97GB | [logs](planet-logs/v0.1.0-planet-linode-50cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-linode-50cpu-128gb.nps) |
+| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB) | 59m cpu:27h6m avg:27.4 | 97GB | [logs](planet-logs/v0.1.0-planet-c5ad-64cpu-128gb.txt) |
 
 ## Alternatives
 

flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java

@@ -86,10 +86,8 @@ class ExternalMergeSort implements FeatureSort {
         "Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory);
     }
     this.workers = workers;
-    this.readerLimit = Math.max(1, config.arguments()
+    this.readerLimit = Math.max(1, config.sortMaxReaders());
-      .getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", 6));
+    this.writerLimit = Math.max(1, config.sortMaxWriters());
-    this.writerLimit = Math.max(1, config.arguments()
-      .getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks", 6));
     LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers);
     try {
       FileUtils.deleteDirectory(dir);

flatmap-core/src/main/java/com/onthegomap/flatmap/config/FlatmapConfig.java

@@ -17,6 +17,8 @@ public record FlatmapConfig(
   boolean emitTilesInOrder,
   boolean forceOverwrite,
   boolean gzipTempStorage,
+  int sortMaxReaders,
+  int sortMaxWriters,
   String nodeMapType,
   String nodeMapStorage,
   String httpUserAgent,
@@ -61,6 +63,10 @@ public record FlatmapConfig(
       arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true),
       arguments.getBoolean("force", "force overwriting output file", false),
       arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
+      arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks",
+        6),
+      arguments.getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks",
+        6),
       arguments
         .getString("nodemap_type", "type of node location map: noop, sortedtable, or sparsearray", "sortedtable"),
       arguments.getString("nodemap_storage", "storage for location map: mmap or ram", "mmap"),

flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmReader.java

@@ -220,6 +220,11 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
     String parseThreadPrefix = "pbfpass2";
     var pipeline = WorkerPipeline.start("osm_pass2", stats)
       .fromGenerator("pbf", osmInputFile.read(parseThreadPrefix, readerThreads))
+      // TODO should use an adaptive batch size to better utilize lots of cpus:
+      //   - make queue size proportional to cores
+      //   - much larger batches when processing points
+      //   - slightly larger batches when processing ways
+      //   - 1_000 is probably fine for relations
       .addBuffer("reader_queue", 50_000, 1_000)
       .<SortableFeature>addWorker("process", processThreads, (prev, next) -> {
         // avoid contention trying to get the thread-local counters by getting them once when thread starts