diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index ab848401b1..8f1071e21c 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -36,7 +36,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "suffix: validations/s" }, "overrides": [] }, @@ -70,9 +70,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: closes/s", "custom": { - "axisLabel": "Events / Sec", + "axisLabel": "Closes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -111,7 +111,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", "spanNulls": 1800000, @@ -158,7 +158,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", "spanNulls": 1800000, @@ -205,7 +205,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", "spanNulls": 1800000, @@ -524,7 +524,14 @@ }, "overrides": [] }, - "id": 12 + "id": 12, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + } }, { "title": "Consensus Accept Duration Heatmap", @@ -539,7 +546,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -584,7 +592,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: rounds/s", "custom": { "axisLabel": "Rounds / Sec", "spanNulls": 1800000, @@ -595,7 +603,14 @@ }, "overrides": [] }, - "id": 14 + "id": 14, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + } }, { "title": "Close Time Vote Bins & Resolution", @@ -657,7 +672,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -880,7 +896,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: failures/s", "custom": { "axisLabel": "Failures / Sec", "spanNulls": 1800000, @@ -927,7 +943,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: checks/s", "custom": { "axisLabel": "Checks / Sec", "spanNulls": 1800000, @@ -968,9 +984,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: changes/s", "custom": { - "axisLabel": "Mode Changes / Sec", + "axisLabel": "Changes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index f598ba5808..16eeb14bee 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -496,9 +496,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -543,9 +543,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -584,9 +584,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -637,9 +637,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -985,8 +985,9 @@ "alignValue": "left", "rowHeight": 0.9, "tooltip": { - "mode": "single", - "sort": "none" + "maxHeight": 600, + "mode": "multi", + "sort": "desc" }, "legend": { "displayMode": "list", @@ -1012,7 +1013,7 @@ "fillOpacity": 80, "lineWidth": 0, "insertNulls": false, - "spanNulls": false + "spanNulls": 1800000 }, "mappings": [ { @@ -1393,16 +1394,49 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", + "unit": "suffix: reads/s", "custom": { - "axisLabel": "Reads/s & Depth", + "axisLabel": "Reads / Sec & Depth", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Queue/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Threads Running/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 24 }, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 3a0997f510..96b2d40c92 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -36,7 +36,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -109,7 +109,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -127,7 +127,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)" @@ -218,9 +219,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Operations / Sec", + "axisLabel": "Transactions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -258,7 +259,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index 7323a32bef..724f05f652 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -63,7 +63,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 24 }, "options": { "tooltip": { @@ -175,9 +175,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -227,9 +227,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -285,9 +285,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -343,9 +343,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -363,7 +363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index a0bb801fe0..ded95f353c 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -419,7 +419,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: fetches/s", "custom": {} }, "overrides": [] @@ -453,7 +453,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: mismatches/s", "thresholds": { "steps": [ { @@ -1886,9 +1886,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "µs", + "unit": "\u00b5s", "custom": { - "axisLabel": "Duration (µs)", + "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1986,9 +1986,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "µs", + "unit": "\u00b5s", "custom": { - "axisLabel": "Wait Time (µs)", + "axisLabel": "Wait Time (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -2140,7 +2140,7 @@ }, { "title": "State Duration Rate (All States)", - "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "type": "timeseries", "gridPos": { "h": 8, @@ -2190,7 +2190,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "short", + "unit": "suffix: s/s", "custom": { "axisLabel": "Rate (s/s)", "spanNulls": 1800000, @@ -2230,9 +2230,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ms", + "unit": "\u00b5s", "custom": { - "axisLabel": "Duration (ms)", + "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -2270,9 +2270,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ms", + "unit": "\u00b5s", "custom": { - "axisLabel": "Wait Time (ms)", + "axisLabel": "Wait Time (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index adc685ce6f..fa812e425a 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -67,9 +67,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -195,9 +195,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Count", + "axisLabel": "Messages / Sec & Bytes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -328,9 +328,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 324aa70e09..47f8cf1a04 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -37,7 +37,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", "spanNulls": 1800000, @@ -77,7 +77,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: validations/s", "custom": { "axisLabel": "Validations / Sec", "spanNulls": 1800000, @@ -111,13 +111,13 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")" + "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")" } ], "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "short" }, "overrides": [] } @@ -144,13 +144,13 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")" + "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")" } ], "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "short" }, "overrides": [] } diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index d4eef4027c..6211e3e254 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -282,7 +282,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: warnings/s", "thresholds": { "steps": [ { @@ -331,7 +331,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: drops/s", "thresholds": { "steps": [ { @@ -380,7 +380,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", "spanNulls": 1800000, @@ -460,7 +460,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", "spanNulls": 1800000, @@ -546,7 +546,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: operations/s", "custom": { "axisLabel": "Operations / Sec", "spanNulls": 1800000, diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 230dcbe0fb..15da043947 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -166,7 +166,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -271,7 +272,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: commands/s", "custom": { "axisLabel": "Commands / Sec", "spanNulls": 1800000, @@ -346,7 +347,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops" + "unit": "suffix: messages/s" }, "overrides": [] }, @@ -385,7 +386,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", "spanNulls": 1800000, @@ -432,7 +433,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", "spanNulls": 1800000, diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index c14b31013d..8092993922 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -36,7 +36,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "thresholds": { "steps": [ { @@ -137,9 +137,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "TX / Sec", + "axisLabel": "Transactions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -183,9 +183,9 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Failed TX / Sec", + "axisLabel": "Transactions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -224,7 +224,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", "spanNulls": 1800000, @@ -271,7 +271,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", "spanNulls": 1800000, @@ -418,7 +418,9 @@ "alignValue": "center", "rowHeight": 0.85, "tooltip": { - "mode": "single" + "mode": "multi", + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "list", @@ -463,7 +465,8 @@ }, "custom": { "fillOpacity": 80, - "lineWidth": 0 + "lineWidth": 0, + "spanNulls": 1800000 } }, "overrides": [] @@ -549,9 +552,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: failures/s", "custom": { - "axisLabel": "Failed Spans / Sec", + "axisLabel": "Failures / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -677,7 +680,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: enqueues/s", "custom": { "axisLabel": "Enqueues / Sec", "spanNulls": 1800000, @@ -800,7 +803,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: cleanups/s", "custom": { "axisLabel": "Cleanups / Sec", "spanNulls": 1800000, diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index b41fbffa7b..edf90ffb90 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1046,7 +1046,7 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: | ---------------------------------- | -------------- | -------------------------------------------------------------------------------------------- | ----------------------------------- | | Transaction Processing Rate | timeseries | `rate(span_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | | Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | -| Transaction Path Distribution | piechart | `sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Path Distribution | piechart | `sum by (local) (increase(span_calls_total{span_name="tx.process"}[5m]))` | `local` | | Transaction Receive vs Suppressed | timeseries | `rate(span_calls_total{span_name="tx.receive"}[5m])` | — | | TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | | TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | @@ -1086,41 +1086,41 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: Requires `trace_peer=1` in the `[telemetry]` config section. -| Panel | Type | PromQL | Labels Used | -| -------------------------------- | ---------- | ------------------------------ | -------------------- | -| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | -| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | -| Proposals Trusted vs Untrusted | piechart | by `proposal_trusted` | `proposal_trusted` | -| Validations Trusted vs Untrusted | piechart | by `validation_trusted` | `validation_trusted` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ------------------------------------------------------------------------- | -------------------- | +| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | +| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | +| Proposals Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `proposal_trusted` | `proposal_trusted` | +| Validations Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `validation_trusted` | `validation_trusted` | ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (All States) | timeseries | `rate(state_accounting__duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | -| Server State | stat | `server_info{metric="server_state"}` | `metric` | -| Uptime | stat | `server_info{metric="uptime"}` | `metric` | -| Peer Count | stat | `server_info{metric="peers"}` | `metric` | -| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | -| Build Version | stat | `build_info` | `version` | -| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | -| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | -| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | --------------------------------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (All States) | timeseries | `rate(state_accounting__duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | ### Network Traffic -- System Metrics (`network-traffic`) @@ -1147,8 +1147,8 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | | Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | | Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | -| Resource Warnings Rate | stat | `rate(warn[5m])` | — | -| Resource Drops Rate | stat | `rate(drop[5m])` | — | +| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | +| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | ### Span → Metric → Dashboard Summary