mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

dashboard ok

pull/1/head
Mike Barry 2021-06-06 15:57:32 -04:00
rodzic 5602bf5123
commit 6d227e690f
2 zmienionych plików z 80 dodań i 50 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

125
grafana.json
Wyświetl plik

@ -16,7 +16,7 @@
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 29, "id": 29,
"iteration": 1622894339083, "iteration": 1623008536252,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -81,13 +81,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_osm_pass1_osm_elements_processed", "expr": "flatmap_osm_pass1_elements_processed_total",
"interval": "", "interval": "",
"legendFormat": "pass1 {{type}}", "legendFormat": "pass1 {{type}}",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_osm_pass2_osm_elements_processed", "expr": "flatmap_osm_pass2_elements_processed_total",
"interval": "", "interval": "",
"legendFormat": "pass2 {{type}}", "legendFormat": "pass2 {{type}}",
"refId": "B" "refId": "B"
@ -144,7 +144,15 @@
"defaults": { "defaults": {
"custom": {} "custom": {}
}, },
"overrides": [] "overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "_running$"
},
"properties": []
}
]
}, },
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -176,22 +184,43 @@
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"seriesOverrides": [], "seriesOverrides": [
{
"alias": "/_running$/",
"linewidth": 0
},
{
"alias": "/.*/",
"fill": 0
},
{
"alias": "/_running$/",
"fill": 1
}
],
"spaceLength": 10, "spaceLength": 10,
"stack": false, "stack": false,
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_osm_pass2_osm_elements_processed\n/\nflatmap_osm_pass1_osm_elements_processed", "expr": "flatmap_osm_pass2_elements_processed_total\n/\nflatmap_osm_pass1_elements_processed_total",
"instant": false,
"interval": "", "interval": "",
"legendFormat": "{{type}}", "legendFormat": "{{type}}",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_mbtiles_features_processed", "expr": "sum(flatmap_mbtiles_features_processed_total) / sum(flatmap_renderer_features_emitted_total)",
"interval": "", "interval": "",
"legendFormat": "mbtiles features", "legendFormat": "mbtiles features",
"refId": "B" "refId": "B"
},
{
"expr": "{__name__=~\"^flatmap_.*running$\"}>0",
"instant": false,
"interval": "",
"legendFormat": "{{__name__}}",
"refId": "C"
} }
], ],
"thresholds": [], "thresholds": [],
@ -283,31 +312,31 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "rate(flatmap_osm_pass1_osm_elements_processed[$interval])", "expr": "rate(flatmap_osm_pass1_elements_processed_total[$interval])",
"interval": "", "interval": "",
"legendFormat": "pass1 {{type}}", "legendFormat": "pass1 {{type}}",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "rate(flatmap_osm_pass2_osm_elements_processed[$interval])", "expr": "rate(flatmap_osm_pass2_elements_processed_total[$interval])",
"interval": "", "interval": "",
"legendFormat": "pass2 {{type}}", "legendFormat": "pass2 {{type}}",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "rate(flatmap_mbtiles_features_processed[$interval])", "expr": "rate(flatmap_mbtiles_features_processed_total[$interval])",
"interval": "", "interval": "",
"legendFormat": "mbtiles features processed", "legendFormat": "mbtiles features processed",
"refId": "C" "refId": "C"
}, },
{ {
"expr": "sum(rate(flatmap_osm_pass2_features_emitted[$interval])) by (layer)", "expr": "sum(rate(flatmap_renderer_features_emitted_total[$interval])) by (layer)",
"interval": "", "interval": "",
"legendFormat": "pass2 emitted {{layer}} feature", "legendFormat": "pass2 emitted {{layer}} feature",
"refId": "D" "refId": "D"
}, },
{ {
"expr": "sum(rate(flatmap_mbtiles_tiles_written_count[$interval])) by (zoom)", "expr": "sum(rate(flatmap_mbtiles_tiles_written_total[$interval])) by (zoom)",
"interval": "", "interval": "",
"legendFormat": "z{{zoom}} tiles written", "legendFormat": "z{{zoom}} tiles written",
"refId": "E" "refId": "E"
@ -822,13 +851,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "(sum(rate(flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"})) * sum(flatmap_osm_pass1_running) > 0", "expr": "(sum(rate(flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"}))",
"interval": "", "interval": "",
"legendFormat": "blocked enqueueing", "legendFormat": "blocked enqueueing",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass1.*$\"}[$interval])) * sum(flatmap_osm_pass1_running) > 0", "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass1.*$\"}[$interval]))",
"interval": "", "interval": "",
"legendFormat": "PBF reader", "legendFormat": "PBF reader",
"refId": "C" "refId": "C"
@ -929,13 +958,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds[$interval]))\n* sum(flatmap_osm_pass1_running) > 0", "expr": "sum(rate(flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "blocked dequeueing", "legendFormat": "blocked dequeueing",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_process.*$\"}[$interval])) * sum(flatmap_osm_pass1_running) > 0", "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_process.*$\"}[$interval]))",
"interval": "", "interval": "",
"legendFormat": "process CPU", "legendFormat": "process CPU",
"refId": "C" "refId": "C"
@ -1030,13 +1059,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count)", "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count)", "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -1131,13 +1160,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -1333,13 +1362,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds", "expr": "flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "enqueue", "legendFormat": "enqueue",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds", "expr": "flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "dequeue", "legendFormat": "dequeue",
"refId": "B" "refId": "B"
@ -1448,7 +1477,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "(sum(rate(flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}))\n* sum(flatmap_osm_pass2_running) > 0", "expr": "(sum(rate(flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}))\n* sum(flatmap_osm_pass2_running) > 0",
"interval": "", "interval": "",
"legendFormat": "blocked enqueueing", "legendFormat": "blocked enqueueing",
"refId": "B" "refId": "B"
@ -1555,13 +1584,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)",
"interval": "", "interval": "",
"legendFormat": "blocked dequeueing", "legendFormat": "blocked dequeueing",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)",
"interval": "", "interval": "",
"legendFormat": "blocked enqueueing", "legendFormat": "blocked enqueueing",
"refId": "B" "refId": "B"
@ -1668,7 +1697,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds[$interval])\n/\nflatmap_osm_pass2_write_threads) * (sum(flatmap_osm_pass2_running) > 0)", "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_write_threads) * (sum(flatmap_osm_pass2_running) > 0)",
"interval": "", "interval": "",
"legendFormat": "blocked dequeueing", "legendFormat": "blocked dequeueing",
"refId": "A" "refId": "A"
@ -1775,13 +1804,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(flatmap_osm_pass2_reader_queue_enqueue_count)", "expr": "sum(flatmap_osm_pass2_reader_queue_enqueue_count_total)",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(flatmap_osm_pass2_reader_queue_dequeue_count)", "expr": "sum(flatmap_osm_pass2_reader_queue_dequeue_count_total)",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -1876,13 +1905,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass2_reader_queue_enqueue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass2_reader_queue_enqueue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -2078,13 +2107,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds", "expr": "flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "enqueue", "legendFormat": "enqueue",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds", "expr": "flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "dequeue", "legendFormat": "dequeue",
"refId": "B" "refId": "B"
@ -2179,13 +2208,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(flatmap_osm_pass2_feature_queue_enqueue_count)", "expr": "sum(flatmap_osm_pass2_feature_queue_enqueue_count_total)",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(flatmap_osm_pass2_feature_queue_dequeue_count)", "expr": "sum(flatmap_osm_pass2_feature_queue_dequeue_count_total)",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -2280,13 +2309,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_count[$interval]))", "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -2482,13 +2511,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds", "expr": "flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "enqueue", "legendFormat": "enqueue",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds", "expr": "flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "dequeue", "legendFormat": "dequeue",
"refId": "B" "refId": "B"
@ -2597,7 +2626,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "rate(flatmap_mbtiles_writer_queue_enqueue_block_time_seconds[$interval])\n/\nflatmap_mbtiles_encoder_threads", "expr": "rate(flatmap_mbtiles_writer_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_encoder_threads",
"interval": "", "interval": "",
"legendFormat": "blocked enqueueing", "legendFormat": "blocked enqueueing",
"refId": "B" "refId": "B"
@ -2704,7 +2733,7 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "rate(flatmap_mbtiles_writer_queue_dequeue_block_time_seconds[$interval])\n/\nflatmap_mbtiles_writer_threads", "expr": "rate(flatmap_mbtiles_writer_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_writer_threads",
"interval": "", "interval": "",
"legendFormat": "blocked dequeueing", "legendFormat": "blocked dequeueing",
"refId": "B" "refId": "B"
@ -2811,13 +2840,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(flatmap_mbtiles_reader_queue_enqueue_count)", "expr": "sum(flatmap_mbtiles_reader_queue_enqueue_count_total)",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(flatmap_mbtiles_reader_queue_dequeue_count)", "expr": "sum(flatmap_mbtiles_reader_queue_dequeue_count_total)",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -2912,13 +2941,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "sum(rate(flatmap_mbtiles_reader_queue_enqueue_count[$interval]))", "expr": "sum(rate(flatmap_mbtiles_reader_queue_enqueue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "emitted", "legendFormat": "emitted",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(flatmap_mbtiles_reader_queue_dequeue_count[$interval]))", "expr": "sum(rate(flatmap_mbtiles_reader_queue_dequeue_count_total[$interval]))",
"interval": "", "interval": "",
"legendFormat": "processed", "legendFormat": "processed",
"refId": "B" "refId": "B"
@ -3114,13 +3143,13 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "flatmap_mbtiles_reader_queue_enqueue_block_time_seconds", "expr": "flatmap_mbtiles_reader_queue_enqueue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "enqueue", "legendFormat": "enqueue",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "flatmap_mbtiles_reader_queue_dequeue_block_time_seconds", "expr": "flatmap_mbtiles_reader_queue_dequeue_block_time_seconds_total",
"interval": "", "interval": "",
"legendFormat": "dequeue", "legendFormat": "dequeue",
"refId": "B" "refId": "B"
@ -3838,12 +3867,12 @@
] ]
}, },
"time": { "time": {
"from": "now-30m", "from": "now-1h",
"to": "now" "to": "now"
}, },
"timepicker": {}, "timepicker": {},
"timezone": "", "timezone": "",
"title": "flatmap", "title": "flatmap",
"uid": "scLmtpaMz", "uid": "scLmtpaMz",
"version": 53 "version": 64
} }

5
src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java
Wyświetl plik

@ -327,16 +327,16 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
private final TileCoord tile; private final TileCoord tile;
private final List<FeatureSort.Entry> entries = new ArrayList<>(); private final List<FeatureSort.Entry> entries = new ArrayList<>();
private LongLongHashMap counts = null; private LongLongHashMap counts = null;
private byte layer = Byte.MAX_VALUE; private byte layer = Byte.MAX_VALUE;
private long numFeaturesProcessed = 0;
public TileFeatures(int tile) { public TileFeatures(int tile) {
this.tile = TileCoord.decode(tile); this.tile = TileCoord.decode(tile);
} }
public long getNumFeatures() { public long getNumFeatures() {
return entries.size(); return numFeaturesProcessed;
} }
public TileCoord coord() { public TileCoord coord() {
@ -395,6 +395,7 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
@Override @Override
public void accept(FeatureSort.Entry entry) { public void accept(FeatureSort.Entry entry) {
numFeaturesProcessed++;
long sortKey = entry.sortKey(); long sortKey = entry.sortKey();
if (extractHasGroupFromSortKey(sortKey)) { if (extractHasGroupFromSortKey(sortKey)) {
byte thisLayer = extractLayerIdFromSortKey(sortKey); byte thisLayer = extractLayerIdFromSortKey(sortKey);