diff --git a/grafana.json b/grafana.json index 458e025b..aa146de2 100644 --- a/grafana.json +++ b/grafana.json @@ -16,7 +16,7 @@ "gnetId": null, "graphTooltip": 0, "id": 29, - "iteration": 1623008536252, + "iteration": 1623028315460, "links": [], "panels": [ { @@ -629,9 +629,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(jvm_gc_collection_seconds_sum{exported_job=\"flatmap\"}[$interval]))", + "expr": "flatmap_bad_input_data_total", "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "{{type}}", "refId": "A" } ], @@ -639,7 +639,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC", + "title": "Data errors", "tooltip": { "shared": true, "sort": 2, @@ -655,7 +655,8 @@ }, "yaxes": [ { - "format": "percentunit", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -798,2413 +799,6 @@ "x": 0, "y": 12 }, - "id": 59, - "panels": [], - "title": "pass1", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum(rate(flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"}))", - "interval": "", - "legendFormat": "blocked enqueueing", - "refId": "B" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass1.*$\"}[$interval]))", - "interval": "", - "legendFormat": "PBF reader", - "refId": "C" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"}[$interval]))* sum(flatmap_osm_pass1_running) > 0", - "interval": "", - "legendFormat": "PBF parsers", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass1 PBF reader", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 62, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total[$interval]))", - "interval": "", - "legendFormat": "blocked dequeueing", - "refId": "B" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_process.*$\"}[$interval]))", - "interval": "", - "legendFormat": "process CPU", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass1 processor", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 63, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass1 reader queue", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 18 - }, - "hiddenSeries": false, - "id": 64, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass1 reader queue/sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass1_reader_queue_capacity", - "interval": "", - "legendFormat": "capacity", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass1_reader_queue_size", - "interval": "", - "legendFormat": "size", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass1 reader queue size", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 18 - }, - "hiddenSeries": false, - "id": 66, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total", - "interval": "", - "legendFormat": "enqueue", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total", - "interval": "", - "legendFormat": "dequeue", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "reader queue block time", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 53, - "panels": [], - "title": "pass2", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 47, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum(rate(flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}))\n* sum(flatmap_osm_pass2_running) > 0", - "interval": "", - "legendFormat": "blocked enqueueing", - "refId": "B" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass2.*$\"}[$interval])) * sum(flatmap_osm_pass2_running) > 0", - "interval": "", - "legendFormat": "PBF reader", - "refId": "C" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}[$interval])) * \nsum(flatmap_osm_pass2_running) > 0", - "interval": "", - "legendFormat": "PBF parsers", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 PBF reader", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 23 - }, - "hiddenSeries": false, - "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "blocked dequeueing", - "refId": "A" - }, - { - "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "blocked enqueueing", - "refId": "B" - }, - { - "expr": "sum(avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_process.*$\"}[$interval]))) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "cpu", - "refId": "C" - }, - { - "expr": "sum(avg(rate(jvm_thread_user_time_seconds_total{name=~\"^osm_pass2_process.*$\"}[$interval]))) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "user time", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 element processor", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 23 - }, - "hiddenSeries": false, - "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_write_threads) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "blocked dequeueing", - "refId": "A" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_write.*$\"}[$interval])) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "cpu", - "refId": "C" - }, - { - "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^osm_pass2_write.*$\"}[$interval])) * (sum(flatmap_osm_pass2_running) > 0)", - "interval": "", - "legendFormat": "user time", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 writer", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 27 - }, - "hiddenSeries": false, - "id": 67, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(flatmap_osm_pass2_reader_queue_enqueue_count_total)", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(flatmap_osm_pass2_reader_queue_dequeue_count_total)", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 reader queue", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 27 - }, - "hiddenSeries": false, - "id": 68, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass2_reader_queue_enqueue_count_total[$interval]))", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_count_total[$interval]))", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 reader features/sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 27 - }, - "hiddenSeries": false, - "id": 69, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass2_reader_queue_capacity", - "interval": "", - "legendFormat": "capacity", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass2_reader_queue_size", - "interval": "", - "legendFormat": "size", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 reader queue size", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 27 - }, - "hiddenSeries": false, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total", - "interval": "", - "legendFormat": "enqueue", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total", - "interval": "", - "legendFormat": "dequeue", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "reader queue block time", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(flatmap_osm_pass2_feature_queue_enqueue_count_total)", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(flatmap_osm_pass2_feature_queue_dequeue_count_total)", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 writer queue count", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 31 - }, - "hiddenSeries": false, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_count_total[$interval]))", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_count_total[$interval]))", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 writer queue/sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass2_feature_queue_capacity", - "interval": "", - "legendFormat": "capacity", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass2_feature_queue_size", - "interval": "", - "legendFormat": "size", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "pass2 writer queue size", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 31 - }, - "hiddenSeries": false, - "id": 70, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total", - "interval": "", - "legendFormat": "enqueue", - "refId": "A" - }, - { - "expr": "flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total", - "interval": "", - "legendFormat": "dequeue", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "writer queue block time", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 57, - "panels": [], - "title": "mbtiles writer pass", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(flatmap_mbtiles_writer_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_encoder_threads", - "interval": "", - "legendFormat": "blocked enqueueing", - "refId": "B" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^mbtiles_encoder.*$\"}[$interval]))", - "interval": "", - "legendFormat": "cpu", - "refId": "C" - }, - { - "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^mbtiles_encoder.*$\"}[$interval]))", - "interval": "", - "legendFormat": "user time", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles encoder", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 36 - }, - "hiddenSeries": false, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(flatmap_mbtiles_writer_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_writer_threads", - "interval": "", - "legendFormat": "blocked dequeueing", - "refId": "B" - }, - { - "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^mbtiles_writer.*$\"}[$interval]))", - "interval": "", - "legendFormat": "cpu", - "refId": "C" - }, - { - "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^mbtiles_writer.*$\"}[$interval]))", - "interval": "", - "legendFormat": "user time", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles writer", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(flatmap_mbtiles_reader_queue_enqueue_count_total)", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(flatmap_mbtiles_reader_queue_dequeue_count_total)", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles features", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 40 - }, - "hiddenSeries": false, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(flatmap_mbtiles_reader_queue_enqueue_count_total[$interval]))", - "interval": "", - "legendFormat": "emitted", - "refId": "A" - }, - { - "expr": "sum(rate(flatmap_mbtiles_reader_queue_dequeue_count_total[$interval]))", - "interval": "", - "legendFormat": "processed", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles features/sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_mbtiles_reader_queue_capacity", - "interval": "", - "legendFormat": "capacity", - "refId": "A" - }, - { - "expr": "flatmap_mbtiles_reader_queue_size", - "interval": "", - "legendFormat": "size", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles queue size", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 40 - }, - "hiddenSeries": false, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "flatmap_mbtiles_reader_queue_enqueue_block_time_seconds_total", - "interval": "", - "legendFormat": "enqueue", - "refId": "A" - }, - { - "expr": "flatmap_mbtiles_reader_queue_dequeue_block_time_seconds_total", - "interval": "", - "legendFormat": "dequeue", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "mbtiles queue block time", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, "id": 14, "panels": [], "title": "Memory", @@ -3228,7 +822,7 @@ "h": 5, "w": 6, "x": 0, - "y": 45 + "y": 13 }, "hiddenSeries": false, "id": 8, @@ -3330,7 +924,7 @@ "h": 5, "w": 6, "x": 6, - "y": 45 + "y": 13 }, "hiddenSeries": false, "id": 10, @@ -3443,7 +1037,7 @@ "h": 5, "w": 6, "x": 12, - "y": 45 + "y": 13 }, "hiddenSeries": false, "id": 6, @@ -3533,7 +1127,7 @@ "h": 5, "w": 6, "x": 18, - "y": 45 + "y": 13 }, "hiddenSeries": false, "id": 20, @@ -3605,6 +1199,2413 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 59, + "panels": [], + "title": "pass1", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"}))", + "interval": "", + "legendFormat": "blocked enqueueing", + "refId": "B" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass1.*$\"}[$interval]))", + "interval": "", + "legendFormat": "PBF reader", + "refId": "C" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_pbf.*$\"}[$interval]))* sum(flatmap_osm_pass1_running) > 0", + "interval": "", + "legendFormat": "PBF parsers", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass1 PBF reader", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 19 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total[$interval]))", + "interval": "", + "legendFormat": "blocked dequeueing", + "refId": "B" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass1_process.*$\"}[$interval]))", + "interval": "", + "legendFormat": "process CPU", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass1 processor", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(flatmap_osm_pass1_reader_queue_enqueue_count_total)", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass1 reader queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 24 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(rate(flatmap_osm_pass1_reader_queue_enqueue_count_total[$interval]))", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass1 reader queue/sec", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass1_reader_queue_capacity", + "interval": "", + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass1_reader_queue_size", + "interval": "", + "legendFormat": "size", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass1 reader queue size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass1_reader_queue_enqueue_block_time_seconds_total", + "interval": "", + "legendFormat": "enqueue", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass1_reader_queue_dequeue_block_time_seconds_total", + "interval": "", + "legendFormat": "dequeue", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "reader queue block time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 53, + "panels": [], + "title": "pass2", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total[$interval]))\n/\ncount(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}))\n* sum(flatmap_osm_pass2_running) > 0", + "interval": "", + "legendFormat": "blocked enqueueing", + "refId": "B" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^pbfpass2.*$\"}[$interval])) * sum(flatmap_osm_pass2_running) > 0", + "interval": "", + "legendFormat": "PBF reader", + "refId": "C" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_pbf.*$\"}[$interval])) * \nsum(flatmap_osm_pass2_running) > 0", + "interval": "", + "legendFormat": "PBF parsers", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 PBF reader", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "blocked dequeueing", + "refId": "A" + }, + { + "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_process_threads) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "blocked enqueueing", + "refId": "B" + }, + { + "expr": "sum(avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_process.*$\"}[$interval]))) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "cpu", + "refId": "C" + }, + { + "expr": "sum(avg(rate(jvm_thread_user_time_seconds_total{name=~\"^osm_pass2_process.*$\"}[$interval]))) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "user time", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 element processor", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 29 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_osm_pass2_write_threads) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "blocked dequeueing", + "refId": "A" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^osm_pass2_write.*$\"}[$interval])) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "cpu", + "refId": "C" + }, + { + "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^osm_pass2_write.*$\"}[$interval])) * (sum(flatmap_osm_pass2_running) > 0)", + "interval": "", + "legendFormat": "user time", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 writer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(flatmap_osm_pass2_reader_queue_enqueue_count_total)", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(flatmap_osm_pass2_reader_queue_dequeue_count_total)", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 reader queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 33 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass2_reader_queue_enqueue_count_total[$interval]))", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(rate(flatmap_osm_pass2_reader_queue_dequeue_count_total[$interval]))", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 reader features/sec", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass2_reader_queue_capacity", + "interval": "", + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass2_reader_queue_size", + "interval": "", + "legendFormat": "size", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 reader queue size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 33 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass2_reader_queue_enqueue_block_time_seconds_total", + "interval": "", + "legendFormat": "enqueue", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass2_reader_queue_dequeue_block_time_seconds_total", + "interval": "", + "legendFormat": "dequeue", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "reader queue block time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(flatmap_osm_pass2_feature_queue_enqueue_count_total)", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(flatmap_osm_pass2_feature_queue_dequeue_count_total)", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 writer queue count", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 37 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_osm_pass2_feature_queue_enqueue_count_total[$interval]))", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(rate(flatmap_osm_pass2_feature_queue_dequeue_count_total[$interval]))", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 writer queue/sec", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass2_feature_queue_capacity", + "interval": "", + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass2_feature_queue_size", + "interval": "", + "legendFormat": "size", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "pass2 writer queue size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 37 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_osm_pass2_feature_queue_enqueue_block_time_seconds_total", + "interval": "", + "legendFormat": "enqueue", + "refId": "A" + }, + { + "expr": "flatmap_osm_pass2_feature_queue_dequeue_block_time_seconds_total", + "interval": "", + "legendFormat": "dequeue", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "writer queue block time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 57, + "panels": [], + "title": "mbtiles writer pass", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(flatmap_mbtiles_writer_queue_enqueue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_encoder_threads", + "interval": "", + "legendFormat": "blocked enqueueing", + "refId": "B" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^mbtiles_encoder.*$\"}[$interval]))", + "interval": "", + "legendFormat": "cpu", + "refId": "C" + }, + { + "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^mbtiles_encoder.*$\"}[$interval]))", + "interval": "", + "legendFormat": "user time", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles encoder", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(flatmap_mbtiles_writer_queue_dequeue_block_time_seconds_total[$interval])\n/\nflatmap_mbtiles_writer_threads", + "interval": "", + "legendFormat": "blocked dequeueing", + "refId": "B" + }, + { + "expr": "avg(rate(jvm_thread_cpu_time_seconds_total{name=~\"^mbtiles_writer.*$\"}[$interval]))", + "interval": "", + "legendFormat": "cpu", + "refId": "C" + }, + { + "expr": "avg(rate(jvm_thread_user_time_seconds_total{name=~\"^mbtiles_writer.*$\"}[$interval]))", + "interval": "", + "legendFormat": "user time", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles writer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 46 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(flatmap_mbtiles_reader_queue_enqueue_count_total)", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(flatmap_mbtiles_reader_queue_dequeue_count_total)", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles features", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 46 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(flatmap_mbtiles_reader_queue_enqueue_count_total[$interval]))", + "interval": "", + "legendFormat": "emitted", + "refId": "A" + }, + { + "expr": "sum(rate(flatmap_mbtiles_reader_queue_dequeue_count_total[$interval]))", + "interval": "", + "legendFormat": "processed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles features/sec", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 46 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_mbtiles_reader_queue_capacity", + "interval": "", + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "flatmap_mbtiles_reader_queue_size", + "interval": "", + "legendFormat": "size", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles queue size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 46 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "flatmap_mbtiles_reader_queue_enqueue_block_time_seconds_total", + "interval": "", + "legendFormat": "enqueue", + "refId": "A" + }, + { + "expr": "flatmap_mbtiles_reader_queue_dequeue_block_time_seconds_total", + "interval": "", + "legendFormat": "dequeue", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mbtiles queue block time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "datasource": null, "gridPos": { @@ -3874,5 +3875,5 @@ "timezone": "", "title": "flatmap", "uid": "scLmtpaMz", - "version": 64 + "version": 66 } diff --git a/src/main/java/com/onthegomap/flatmap/monitoring/PrometheusStats.java b/src/main/java/com/onthegomap/flatmap/monitoring/PrometheusStats.java index 0c6c052e..fe498548 100644 --- a/src/main/java/com/onthegomap/flatmap/monitoring/PrometheusStats.java +++ b/src/main/java/com/onthegomap/flatmap/monitoring/PrometheusStats.java @@ -9,8 +9,10 @@ import io.prometheus.client.GaugeMetricFamily; import io.prometheus.client.Histogram; import io.prometheus.client.exporter.BasicAuthHttpConnectionFactory; import io.prometheus.client.exporter.PushGateway; +import io.prometheus.client.exporter.common.TextFormat; import io.prometheus.client.hotspot.DefaultExports; import java.io.IOException; +import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.net.URL; @@ -37,24 +39,26 @@ public class PrometheusStats implements Stats { private final CollectorRegistry registry = new CollectorRegistry(); private final Timers timers = new Timers(); - private static final String NAMESPACE = "flatmap"; private static final String BASE = "flatmap_"; - private final PushGateway pg; - private final ScheduledExecutorService executor; + private PushGateway pg; + private ScheduledExecutorService executor; private final String job; private final Map filesToMonitor = Collections.synchronizedMap(new LinkedHashMap<>()); private final Map heapObjectsToMonitor = Collections .synchronizedMap(new LinkedHashMap<>()); - public PrometheusStats(String destination, String job, Duration interval) { + public PrometheusStats(String job) { this.job = job; - try { - DefaultExports.register(registry); - new ThreadDetailsExports().register(registry); - new InProgressTasks().register(registry); - new FileSizeCollector().register(registry); - new HeapObjectSizeCollector().register(registry); + DefaultExports.register(registry); + new ThreadDetailsExports().register(registry); + new InProgressTasks().register(registry); + new FileSizeCollector().register(registry); + new HeapObjectSizeCollector().register(registry); + } + public PrometheusStats(String destination, String job, Duration interval) { + this(job); + try { URL url = new URL(destination); pg = new PushGateway(url); if (url.getUserInfo() != null) { @@ -134,6 +138,15 @@ public class PrometheusStats implements Stats { emittedFeatures.labels(Integer.toString(z), layer).inc(number); } + public String getMetricsAsString() { + try (StringWriter writer = new StringWriter()) { + TextFormat.write004(writer, registry.metricFamilySamples()); + return writer.toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + private final Histogram tilesWrittenBytes = Histogram .build(BASE + "mbtiles_tile_written_bytes", "Written tile sizes by zoom level") .buckets(1_000, 10_000, 100_000, 500_000) diff --git a/src/test/java/com/onthegomap/flatmap/monitoring/CounterTest.java b/src/test/java/com/onthegomap/flatmap/monitoring/CounterTest.java new file mode 100644 index 00000000..209357f9 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/monitoring/CounterTest.java @@ -0,0 +1,57 @@ +package com.onthegomap.flatmap.monitoring; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +public class CounterTest { + + @Test + public void testSingleThreadedCounter() { + var counter = Counter.newSingleThreadCounter(); + assertEquals(0, counter.get()); + + counter.inc(); + assertEquals(1, counter.get()); + counter.incBy(2); + assertEquals(3, counter.get()); + counter.incBy(-1); + assertEquals(2, counter.get()); + } + + @Test + public void testMultiThreadedCounter() throws InterruptedException { + var counter = Counter.newMultiThreadCounter(); + Thread t1 = new Thread(() -> { + counter.incBy(1); + counter.incBy(1); + }); + t1.start(); + Thread t2 = new Thread(() -> { + counter.incBy(1); + }); + t2.start(); + t1.join(); + t2.join(); + assertEquals(3, counter.get()); + } + + @Test + public void testMultiThreadedSubCounter() throws InterruptedException { + var counter = Counter.newMultiThreadCounter(); + Thread t1 = new Thread(() -> { + var subCounter = counter.counterForThread(); + subCounter.incBy(1); + subCounter.incBy(1); + }); + t1.start(); + Thread t2 = new Thread(() -> { + var subCounter = counter.counterForThread(); + subCounter.incBy(1); + }); + t2.start(); + t1.join(); + t2.join(); + assertEquals(3, counter.get()); + } +} diff --git a/src/test/java/com/onthegomap/flatmap/monitoring/PrometheusStatsTest.java b/src/test/java/com/onthegomap/flatmap/monitoring/PrometheusStatsTest.java new file mode 100644 index 00000000..2ccdb599 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/monitoring/PrometheusStatsTest.java @@ -0,0 +1,154 @@ +package com.onthegomap.flatmap.monitoring; + +import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.io.TempDir; + +public class PrometheusStatsTest { + + @TestFactory + public Stream testInitialStat() { + PrometheusStats stats = new PrometheusStats("job"); + String metrics = stats.getMetricsAsString(); + return testContains(metrics, + "^jvm_thread_cpu_time_seconds_total\\{", + "^jvm_thread_user_time_seconds_total\\{", + "^jvm_system_load_avg ", + "^jvm_available_processors [0-9\\.]+$" + ); + } + + @Test + public void testTimer() { + PrometheusStats stats = new PrometheusStats("job"); + stats.time("task1", () -> { + assertContainsStat("^flatmap_task1_running 1", stats); + assertContainsStat("^flatmap_task1_elapsed_time_seconds [0-9\\.]+$", stats); + assertContainsStat("^flatmap_task1_cpu_time_seconds [0-9\\.]+$", stats); + }); + assertContainsStat("^flatmap_task1_running 0", stats); + assertContainsStat("^flatmap_task1_elapsed_time_seconds [0-9\\.]+$", stats); + assertContainsStat("^flatmap_task1_cpu_time_seconds [0-9\\.]+$", stats); + + assertFalse(stats.timers().all().get("task1").running()); + } + + @Test + public void testGauge() { + PrometheusStats stats = new PrometheusStats("job"); + stats.gauge("gauge1", 1); + stats.gauge("gauge2", () -> 2); + assertContainsStat("^flatmap_gauge1 1", stats); + assertContainsStat("^flatmap_gauge2 2", stats); + } + + @Test + public void testProcessedElement() { + PrometheusStats stats = new PrometheusStats("job"); + stats.processedElement("type1", "layer1"); + stats.processedElement("type1", "layer1"); + stats.processedElement("type1", "layer2"); + assertContainsStat("^flatmap_renderer_elements_processed_total\\{.*layer1.* 2", stats); + assertContainsStat("^flatmap_renderer_elements_processed_total\\{.*layer2.* 1", stats); + } + + @Test + public void testDataError() { + PrometheusStats stats = new PrometheusStats("job"); + stats.dataError("err1"); + stats.dataError("err1"); + stats.dataError("err2"); + assertContainsStat("^flatmap_bad_input_data_total\\{.*err1.* 2", stats); + assertContainsStat("^flatmap_bad_input_data_total\\{.*err2.* 1", stats); + } + + @Test + public void testEmittedFeatures() { + PrometheusStats stats = new PrometheusStats("job"); + stats.emittedFeatures(0, "layer1", 2); + stats.emittedFeatures(0, "layer1", 2); + stats.emittedFeatures(0, "layer2", 1); + assertContainsStat("^flatmap_renderer_features_emitted_total\\{.*layer1.* 4", stats); + assertContainsStat("^flatmap_renderer_features_emitted_total\\{.*layer2.* 1", stats); + } + + @Test + public void testWroteTile() { + PrometheusStats stats = new PrometheusStats("job"); + stats.wroteTile(0, 10); + stats.wroteTile(0, 10_000); + assertContainsStat("^flatmap_mbtiles_tile_written_bytes_bucket\\{.*le=\"1000\\..* 1", stats); + assertContainsStat("^flatmap_mbtiles_tile_written_bytes_bucket\\{.*le=\"10000\\..* 2", stats); + } + + @Test + public void testMonitorFile(@TempDir Path path) throws IOException { + PrometheusStats stats = new PrometheusStats("job"); + stats.monitorFile("test", path); + assertContainsStat("^flatmap_file_test_size_bytes 0", stats); + + Files.writeString(path.resolve("data"), "abc"); + assertContainsStat("^flatmap_file_test_size_bytes [0-9]", stats); + } + + @Test + public void testMonitorInMemoryObject() { + PrometheusStats stats = new PrometheusStats("job"); + stats.monitorInMemoryObject("test", () -> 10); + assertContainsStat("^flatmap_heap_object_test_size_bytes 10", stats); + } + + private static Counter.Readable counterAt(int num) { + var result = Counter.newSingleThreadCounter(); + result.incBy(num); + return result; + } + + @Test + public void testCounter() { + PrometheusStats stats = new PrometheusStats("job"); + stats.counter("counter1", () -> 1); + stats.counter("counter2", "label", () -> Map.of( + "value1", counterAt(1), + "value2", counterAt(2) + )); + stats.longCounter("long").incBy(100); + stats.nanoCounter("nanos").incBy((long) (NANOSECONDS_PER_SECOND / 2)); + assertContainsStat("^flatmap_counter1_total 1", stats); + assertContainsStat("^flatmap_counter2_total\\{.*label=\"value1\".* 1", stats); + assertContainsStat("^flatmap_counter2_total\\{.*label=\"value2\".* 2", stats); + assertContainsStat("^flatmap_long_total 100", stats); + assertContainsStat("^flatmap_nanos_total 0.5", stats); + } + + private static Stream testContains(String stats, String... regexes) { + return Stream.of(regexes).map(re -> dynamicTest(re, () -> assertContainsStat(re, stats))); + } + + private static void assertContainsStat(String regex, PrometheusStats stats) { + assertContainsStat(regex, stats.getMetricsAsString()); + } + + private static void assertContainsStat(String regex, String stats) { + Pattern pattern = Pattern.compile(regex); + for (String line : stats.split("\n")) { + if (!line.startsWith("#") && pattern.matcher(line).find()) { + return; + } + } + System.err.println(stats); + fail("could not find " + regex); + } +} diff --git a/src/test/java/com/onthegomap/flatmap/monitoring/TimerTest.java b/src/test/java/com/onthegomap/flatmap/monitoring/TimerTest.java new file mode 100644 index 00000000..9f769e61 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/monitoring/TimerTest.java @@ -0,0 +1,26 @@ +package com.onthegomap.flatmap.monitoring; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TimerTest { + + @Test + public void testTimer() { + Timer timer = new Timer().start(); + ProcessTime elapsed1 = timer.elapsed(); + ProcessTime elapsed2 = timer.stop().elapsed(); + ProcessTime elapsed3 = timer.elapsed(); + + assertEquals(elapsed2.wall(), elapsed3.wall()); + assertLessThan(elapsed1.wall(), elapsed2.wall()); + assertFalse(elapsed3.cpu().isEmpty(), "no CPU time"); + } + + private > void assertLessThan(T a, T b) { + assertTrue(a.compareTo(b) < 0, a + " is not less than " + b); + } +} diff --git a/src/test/java/com/onthegomap/flatmap/monitoring/TimersTest.java b/src/test/java/com/onthegomap/flatmap/monitoring/TimersTest.java new file mode 100644 index 00000000..e29afc83 --- /dev/null +++ b/src/test/java/com/onthegomap/flatmap/monitoring/TimersTest.java @@ -0,0 +1,29 @@ +package com.onthegomap.flatmap.monitoring; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TimersTest { + + @Test + public void testTimers() { + Timers timers = new Timers(); + assertTrue(timers.all().isEmpty()); + timers.printSummary(); + + timers.time("task1", () -> { + assertTrue(timers.all().get("task1").running()); + }); + + assertFalse(timers.all().get("task1").running()); + + var finish = timers.startTimer("task2"); + assertTrue(timers.all().get("task2").running()); + finish.stop(); + assertFalse(timers.all().get("task2").running()); + + timers.printSummary(); + } +}