Files
rippled/docker/telemetry/grafana/dashboards/rpc-performance.json
Pratik Mankawde 97089a3b6f fix(telemetry): stop repeating panels stranding their neighbours
A repeating panel expands into one copy per network at view time and, with
maxPerRow=2, claims the whole row. The non-repeating panel paired beside it was
pushed to the next line but kept its stored x=12, so it rendered on the right
against an empty gap.

Two changes to how the layout is planned:
  - a repeating panel gets a row to itself. It keeps w=12, so its copies still
    tile two across inside that row.
  - single-value panels are grouped to the top of each row section, so the
    charts that follow pair with each other instead of being split up by an
    interleaved repeat. Without this the gaps just become wasted half-rows.

Verified per dashboard: no panel lost, every targets block byte-identical, ids
1..N, no overlaps, and no row left with a gap on its left.
2026-08-07 12:54:25 +01:00

604 lines
22 KiB
JSON

{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"title": "RPC Error Rate [$xrpl_network_type]",
"description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "bargauge",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 0
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])) * 100",
"interval": "15s",
"legendFormat": "{{command}} [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": {
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"repeat": "xrpl_network_type",
"repeatDirection": "h",
"maxPerRow": 2,
"id": 1
},
{
"title": "Top Commands by Volume [$xrpl_network_type]",
"description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "bargauge",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 10
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))",
"interval": "15s",
"legendFormat": "{{command}} [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"repeat": "xrpl_network_type",
"repeatDirection": "h",
"maxPerRow": 2,
"id": 2
},
{
"title": "WebSocket Message Rate [$xrpl_network_type]",
"description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession",
"type": "stat",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 20
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "WS Messages / Sec [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: messages/s"
},
"overrides": []
},
"repeat": "xrpl_network_type",
"repeatDirection": "h",
"maxPerRow": 2,
"id": 3
},
{
"title": "RPC Request Rate by Command",
"description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 30
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "{{command}} [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"axisLabel": "Requests / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 4
},
{
"title": "RPC Latency P95 by Command",
"description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 30
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))",
"interval": "15s",
"legendFormat": "P95 {{command}} [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"axisLabel": "Latency (ms)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 5
},
{
"title": "RPC Latency Heatmap",
"description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "heatmap",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 40
},
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"mode": "scheme",
"scheme": "Turbo",
"steps": 64
},
"legend": {
"show": true
},
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
},
"yAxis": {
"axisLabel": "Duration (ms)",
"unit": "ms"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum(increase(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])) by (le)",
"interval": "15s",
"legendFormat": "{{le}}",
"format": "heatmap"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"spanNulls": 1800000
}
}
},
"id": 6
},
{
"title": "Overall RPC Throughput",
"description": "**What:** Aggregate request flow through two pipeline layers: the outer HTTP handler that accepts connections and the inner layer that parses and dispatches.\n**How it's computed:** Per-second rate of each layer over a 5-minute window, per node.\n**Reading it:** The two lines should overlap when requests flow cleanly.\n**Healthy range:** Workload-dependent; both layers tracking each other.\n**Watch for:** A gap means requests are being queued or rejected before dispatch.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession, ServerHandler::processRequest",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 40
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.http_request\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "rpc.http_request / Sec [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.process\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "rpc.process / Sec [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"axisLabel": "Requests / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 7
},
{
"title": "RPC Success vs Error",
"description": "**What:** Aggregate rate of successful versus failed RPC commands across all types.\n**How it's computed:** Per-second rate of each outcome over a 5-minute window, per node.\n**Reading it:** Success should dominate; the error line should stay low.\n**Healthy range:** Error rate near zero; success rate is workload-dependent.\n**Watch for:** A sustained rise in the error line warrants drilling into the per-command breakdown.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 50
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_UNSET\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "Success [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "Error [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: commands/s",
"custom": {
"axisLabel": "Commands / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 8
},
{
"title": "RPC Resource Cost by Command",
"description": "**What:** RPC traffic grouped by its resource-cost category, distinguishing cheap lookups from expensive or malformed requests.\n**How it's computed:** Per-second rate per cost category over a 5-minute window.\n**Reading it:** Low-cost categories should dominate.\n**Healthy range:** Mostly low-cost traffic; workload-dependent.\n**Watch for:** A rising share of high-cost or malformed categories points to problematic or abusive clients.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 60
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "lastNotNull"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (load_type) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=~\"rpc.command.*\", load_type!=\"\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "{{load_type}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: requests/s",
"custom": {
"axisLabel": "Requests / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 9
},
{
"title": "Batch vs Single RPC Requests",
"description": "**What:** Rate of batched RPC requests versus single requests.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** Most traffic is typically single requests.\n**Healthy range:** Workload-dependent; batch share usually small.\n**Watch for:** A high batch rate may indicate bulk-automation clients or an attempt to amplify load per connection.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processRequest",
"type": "timeseries",
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 70
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc",
"maxHeight": 600
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"rpc.process\", is_batch=\"true\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "Batch [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"rpc.process\", is_batch=\"false\"}[$__rate_interval]))",
"interval": "15s",
"legendFormat": "Single [{{service_instance_id}}]"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: requests/s",
"custom": {
"axisLabel": "Requests / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 5,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none"
}
},
"overrides": []
},
"id": 10
}
],
"schemaVersion": 39,
"tags": ["rpc"],
"templating": {
"list": [
{
"name": "service_name",
"label": "Service Name",
"description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
"type": "query",
"query": "label_values(service_name)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "xrpld",
"value": "xrpld"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "deployment_environment",
"label": "Environment",
"description": "Filter by deployment tier (local/ci/test/prod)",
"type": "query",
"query": "label_values(deployment_environment)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_network_type",
"label": "Network",
"description": "Filter by XRPL network (mainnet/testnet/devnet)",
"type": "query",
"query": "label_values(xrpl_network_type)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "node",
"label": "Node",
"description": "Filter by rippled node (service.instance.id — e.g. Node-1)",
"type": "query",
"query": "label_values(target_info, service_instance_id)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "command",
"label": "RPC Command",
"description": "Filter by RPC command name (e.g., server_info, submit)",
"type": "query",
"query": "label_values(span_calls_total{span_name=~\"rpc.command.*\"}, command)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"title": "RPC Performance",
"uid": "rpc-performance",
"description": "Client RPC service health: per-command request rates, latency, error rates, throughput across pipeline layers, and resource cost."
}