diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 0c127ddcb3..8bda530684 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -87,6 +87,43 @@ "panels": [], "id": 3 }, + { + "title": "Validation Send Rate [$xrpl_network_type]", + "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 10 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: validations/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Consensus Proposals Sent Rate", "description": "**What:** How often this node broadcasts its own consensus proposal (its candidate transaction set) to the network.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** A proposing validator emits a steady stream, roughly one per consensus round; observing-only nodes emit none.\n**Healthy range:** About one proposal per round (~0.2-0.3/s on mainnet) for a proposing node.\n**Watch for:** A validator that drops to zero has stopped proposing and may be unhealthy or misconfigured.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::propose", @@ -95,7 +132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -130,7 +167,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Close Duration", @@ -140,7 +177,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -175,43 +212,6 @@ }, "overrides": [] }, - "id": 5 - }, - { - "title": "Validation Send Rate [$xrpl_network_type]", - "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 18 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: validations/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 6 }, { diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 3532137aef..7de2692b78 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -9,6 +9,67 @@ "id": null, "links": [], "panels": [ + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Ledger Data \u2014 Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", @@ -17,7 +78,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -57,7 +118,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Ledger Data \u2014 Transaction", @@ -67,7 +128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -119,7 +180,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Ledger Data \u2014 Account State", @@ -129,7 +190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -169,7 +230,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Ledger Traffic \u2014 Ledger", @@ -179,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -219,7 +280,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Traffic \u2014 Transaction", @@ -229,7 +290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -281,7 +342,7 @@ }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Ledger Traffic \u2014 Account State", @@ -291,7 +352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -331,7 +392,7 @@ }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "GetObject \u2014 Ledger", @@ -341,7 +402,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -381,7 +442,7 @@ }, "overrides": [] }, - "id": 7 + "id": 8 }, { "title": "GetObject \u2014 Transaction", @@ -391,7 +452,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -443,7 +504,7 @@ }, "overrides": [] }, - "id": 8 + "id": 9 }, { "title": "GetObject \u2014 Account State", @@ -453,7 +514,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -493,7 +554,7 @@ }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "GetObject Messages \u2014 Ledger", @@ -503,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -537,7 +598,7 @@ }, "overrides": [] }, - "id": 10 + "id": 11 }, { "title": "GetObject Messages \u2014 Transaction", @@ -547,7 +608,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -587,7 +648,7 @@ }, "overrides": [] }, - "id": 11 + "id": 12 }, { "title": "GetObject Messages \u2014 Account State", @@ -597,7 +658,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -631,7 +692,7 @@ }, "overrides": [] }, - "id": 12 + "id": 13 }, { "title": "GetObject Messages \u2014 Specials", @@ -641,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -687,7 +748,7 @@ }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "GetObject \u2014 Specials", @@ -697,7 +758,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -767,68 +828,7 @@ }, "overrides": [] }, - "id": 14 - }, - { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 56 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "displayMode": "gradient", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "decbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1048576 - }, - { - "color": "red", - "value": 104857600 - } - ] - } - }, - "overrides": [] - }, - "id": 15, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 15 }, { "title": "Sync Diagnostics", diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 4c04a97c66..e4b8f02b3d 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -58,6 +58,43 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Validation Rate [$xrpl_network_type]", + "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: ledgers/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, { "title": "Ledger Build Duration", "description": "**What:** Time to build a full ledger, including applying transactions, writing state, and finalising acceptance.\n**How it's computed:** 95th and 50th percentile of per-build durations over a 5-minute window, per node.\n**Reading it:** Scales with transaction volume and disk I/O.\n**Healthy range:** Tens to a few hundred milliseconds under typical load.\n**Watch for:** Long build times indicate expensive transaction sets or I/O pressure while flushing state.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", @@ -109,43 +146,6 @@ }, "overrides": [] }, - "id": 3 - }, - { - "title": "Ledger Validation Rate [$xrpl_network_type]", - "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 13 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: ledgers/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 4 }, { @@ -155,8 +155,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 17 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -197,7 +197,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 13 }, "collapsed": false, "panels": [], @@ -211,7 +211,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -264,7 +264,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -308,7 +308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 22 }, "collapsed": false, "panels": [], @@ -322,7 +322,7 @@ "h": 4, "w": 12, "x": 0, - "y": 35 + "y": 23 }, "options": { "tooltip": { @@ -359,7 +359,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 27 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index f0a560c73a..0e11e71d9d 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -113,6 +113,90 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Fetch Rate [$xrpl_network_type]", + "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Fetches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: fetches/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, + { + "title": "Ledger History Mismatches [$xrpl_network_type]", + "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: mismatches/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Operating Mode (Time Share)", "description": "**What:** Fraction of time in each operating mode (per-interval): Disconnected, Connected, Syncing, Tracking, Full.\n**How it's computed:** Per-mode duration rate divided by the sum of all five mode rates over the interval; the five shares stack to 100%.\n**Reading it:** Full should fill almost the entire band on a healthy node.\n**Healthy range:** Full near 100%; other modes near 0.\n**Watch for:** A rising share of Syncing, Connected, or Disconnected (instability or repeated resync).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", @@ -121,7 +205,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -186,7 +270,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Operating Mode Transitions", @@ -196,7 +280,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -258,7 +342,7 @@ }, "overrides": [] }, - "id": 4 + "id": 6 }, { "title": "I/O Latency", @@ -268,7 +352,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -309,7 +393,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "Job Queue Depth", @@ -319,7 +403,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -354,90 +438,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Ledger Fetch Rate [$xrpl_network_type]", - "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Fetches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: fetches/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "Ledger History Mismatches [$xrpl_network_type]", - "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: mismatches/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.01 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { @@ -453,6 +453,112 @@ "panels": [], "id": 9 }, + { + "title": "FullBelowCache Hit Rate [$xrpl_network_type]", + "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", + "type": "gauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Hit Rate" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 + }, + { + "title": "Ledger Publish Gap [$xrpl_network_type]", + "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Publish Gap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 + }, { "title": "Key Jobs Execution Time", "description": "**What:** Execution time of the most important job types (accept/advance ledger, transaction, write objects, heartbeat, sweep, trusted validation/proposal, publish, client RPC, ledger data) at the selected quantile.\n**How it's computed:** Selected quantile of each job's execution-time histogram over 5 minutes.\n**Reading it:** Lower and stable is better; shows where consensus, transaction, and maintenance time goes.\n**Healthy range:** workload-dependent; most jobs in low tens of milliseconds.\n**Watch for:** One job type spiking (a specific bottleneck) or broad increases (overload).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", @@ -461,7 +567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -565,7 +671,7 @@ }, "overrides": [] }, - "id": 10 + "id": 12 }, { "title": "Key Jobs Dequeue Wait Time", @@ -575,7 +681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -679,7 +785,7 @@ }, "overrides": [] }, - "id": 11 + "id": 13 }, { "title": "FullBelowCache Size", @@ -689,7 +795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 37 }, "options": { "tooltip": { @@ -723,112 +829,6 @@ }, "overrides": [] }, - "id": 12 - }, - { - "title": "FullBelowCache Hit Rate [$xrpl_network_type]", - "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", - "type": "gauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Hit Rate" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 25 - }, - { - "color": "green", - "value": 50 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 13 - }, - { - "title": "Ledger Publish Gap [$xrpl_network_type]", - "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Publish Gap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 14 }, { @@ -838,8 +838,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 45 + "x": 12, + "y": 37 }, "options": { "tooltip": { @@ -889,7 +889,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 45 }, "options": { @@ -933,8 +933,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 45 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 094f864531..32c7868e93 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -247,112 +247,6 @@ "panels": [], "id": 6 }, - { - "title": "Pathfinding Fast Duration", - "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 7 - }, - { - "title": "Pathfinding Full Duration", - "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 8 - }, { "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "**What:** Rate of resource-limit warnings raised when a peer or client exceeds its usage warning threshold.\n**How it's computed:** Per-second rate of the warn counter over 5 minutes.\n**Reading it:** Occasional warnings are normal under load; a rising rate flags aggressive clients.\n**Healthy range:** workload-dependent; low.\n**Watch for:** A climbing warning rate (clients approaching limits, a precursor to drops).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", @@ -361,7 +255,7 @@ "h": 4, "w": 12, "x": 0, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -404,7 +298,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 9 + "id": 7 }, { "title": "Resource Drops Rate [$xrpl_network_type]", @@ -414,7 +308,7 @@ "h": 4, "w": 12, "x": 12, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -457,6 +351,112 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 8 + }, + { + "title": "Pathfinding Fast Duration", + "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Pathfinding Full Duration", + "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, "id": 10 }, { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index f2f19ce578..aa8bac5c83 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -8,96 +8,6 @@ "id": null, "links": [], "panels": [ - { - "title": "RPC Request Rate by Command", - "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { - "axisLabel": "Requests / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 1 - }, - { - "title": "RPC Latency P95 by Command", - "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 {{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Latency (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 2 - }, { "title": "RPC Error Rate [$xrpl_network_type]", "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -106,7 +16,7 @@ "h": 4, "w": 12, "x": 0, - "y": 8 + "y": 0 }, "options": { "tooltip": { @@ -149,8 +59,172 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 1 + }, + { + "title": "Top Commands by Volume [$xrpl_network_type]", + "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 + }, + { + "title": "WebSocket Message Rate [$xrpl_network_type]", + "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: messages/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, "id": 3 }, + { + "title": "RPC Request Rate by Command", + "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "RPC Latency P95 by Command", + "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 {{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 5 + }, { "title": "RPC Latency Heatmap", "description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -159,7 +233,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -191,7 +265,7 @@ } } }, - "id": 4 + "id": 6 }, { "title": "Overall RPC Throughput", @@ -201,7 +275,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -244,7 +318,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "RPC Success vs Error", @@ -254,7 +328,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 24 }, "options": { "tooltip": { @@ -297,80 +371,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Top Commands by Volume [$xrpl_network_type]", - "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "WebSocket Message Rate [$xrpl_network_type]", - "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: messages/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 27e7254194..565edda59a 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -8,6 +8,60 @@ "id": null, "links": [], "panels": [ + { + "title": "Transaction Apply Failed Rate [$xrpl_network_type]", + "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Failed / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: transactions/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 + }, { "title": "Transaction Processing Rate", "description": "**What:** Rate of transactions entering the processing pipeline (submitted locally or relayed by peers) versus raw transaction messages arriving from peers before deduplication.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The received line sits above the processed line by the volume of duplicates filtered out.\n**Healthy range:** Workload-dependent; scales with network transaction volume.\n**Watch for:** A large and growing gap means heavy duplicate traffic; a processed-rate collapse means submissions are not being handled.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction, src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", @@ -16,7 +70,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -59,7 +113,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Transaction Processing Latency by Type", @@ -69,7 +123,7 @@ "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -117,7 +171,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Transaction Path Distribution", @@ -127,7 +181,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -151,7 +205,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Transaction Receive vs Suppressed", @@ -161,7 +215,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -196,7 +250,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Transaction Processing Duration Heatmap", @@ -206,7 +260,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -238,7 +292,7 @@ } } }, - "id": 5 + "id": 6 }, { "title": "Transaction Apply Duration per Ledger", @@ -248,7 +302,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -291,60 +345,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Transaction Apply Failed Rate [$xrpl_network_type]", - "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 32 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Failed / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: transactions/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 7 }, {