From 92f9b88a217fc6d8fe6ac61f6c3d32ffa4048329 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:39:58 +0100 Subject: [PATCH] style(telemetry): uniform layout and stable panel ids for phase-9 dashboards Applies the same normalization the phase-7-owned dashboards received, to the five introduced on this branch: fee-market, job-queue, log-derived-insights, peer-quality and validator-health. - id written as 1..N so panelId deep links address a specific panel rather than whichever panel happens to sit in that position - gridPos at most two panels across; charts h=8, stats/gauges h=4, tables/logs h=12 full width. Panels pair only with an equal-height neighbour, so no row keeps a ragged empty cell. validator-health and peer-quality were stacked single-file at full width, which is why they were the longest to scroll. - line charts lineWidth=1, fillOpacity=0, pointSize=5, gradientMode=none - repeat xrpl_network_type (horizontal, maxPerRow=2) with a [$xrpl_network_type] title suffix on stat/gauge/bargauge/ table/state-timeline panels, where two networks overlaid in one panel reads as noise. Line charts keep their networks as separate series. - decimals 0 on panels counting discrete things (peers, quorum, connection mix) - a fractional peer count is meaningless. - rows category rows added to fee-market, job-queue and peer-quality Panels with a right-hand legend stay full width; a side legend needs the room. Raw-text edits, so untouched lines keep their formatting and escaping byte for byte. Verified per dashboard against origin/phase9: panel count unchanged, no panel lost, every targets block byte-identical, descriptions unchanged, ids exactly 1..N, and no two panels overlapping on the grid. --- .../grafana/dashboards/fee-market.json | 133 +++++-- .../grafana/dashboards/job-queue.json | 136 +++++-- .../dashboards/log-derived-insights.json | 354 ++++++++++-------- .../grafana/dashboards/peer-quality.json | 134 +++++-- .../grafana/dashboards/validator-health.json | 262 ++++++++----- 5 files changed, 663 insertions(+), 356 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 8d1cc6bfd8..ab4b502f04 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -39,6 +39,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Queue Depth & Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Transaction Queue Depth", "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", @@ -47,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -79,16 +92,20 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 2 }, { "title": "Transactions Per Ledger", @@ -98,7 +115,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -130,16 +147,32 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 3 + }, + { + "title": "Fee Escalation", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 4 }, { "title": "Fee Escalation Levels", @@ -147,9 +180,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -195,20 +228,23 @@ "custom": { "axisLabel": "Fee Level", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "scaleDistribution": { "type": "log", "log": 2 }, - "spanNulls": 1800000 + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Load Factor Breakdown", @@ -217,8 +253,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -264,9 +300,11 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -289,7 +327,8 @@ } }, "overrides": [] - } + }, + "id": 6 }, { "title": "Load Factor Components", @@ -298,8 +337,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 18 }, "options": { "tooltip": { @@ -338,16 +377,32 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 7 + }, + { + "title": "Queue Rejections", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "collapsed": false, + "panels": [], + "id": 8 }, { "title": "Queue Abandonment Rate (Expired)", @@ -357,7 +412,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -384,11 +439,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Queue Admission Rejections (Dropped)", @@ -398,7 +457,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -425,11 +484,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index c7e745e7e9..f509b39f60 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -10,14 +10,27 @@ "links": [], "panels": [ { - "title": "Current Job Latency (p99 Gauge)", + "title": "Current State", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, + { + "title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]", "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", "type": "gauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "reduceOptions": { @@ -68,7 +81,23 @@ }, "overrides": [] }, - "id": 1 + "id": 2, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "panels": [], + "id": 3 }, { "title": "Job Throughput Rate (Per Second)", @@ -76,9 +105,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 6 }, "options": { "tooltip": { @@ -113,13 +142,14 @@ "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -127,7 +157,7 @@ }, "overrides": [] }, - "id": 2 + "id": 4 }, { "title": "Per-Job-Type Queued Rate", @@ -137,7 +167,7 @@ "h": 8, "w": 24, "x": 0, - "y": 16 + "y": 14 }, "options": { "tooltip": { @@ -166,12 +196,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -179,7 +210,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Per-Job-Type Finish Rate", @@ -189,7 +220,7 @@ "h": 8, "w": 24, "x": 0, - "y": 24 + "y": 22 }, "options": { "tooltip": { @@ -218,12 +249,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -231,7 +263,20 @@ }, "overrides": [] }, - "id": 4 + "id": 6 + }, + { + "title": "Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Job Queue Wait Time", @@ -241,7 +286,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -270,13 +315,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -284,7 +330,7 @@ }, "overrides": [] }, - "id": 5 + "id": 8 }, { "title": "Job Execution Time", @@ -294,7 +340,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -323,13 +369,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -337,7 +384,7 @@ }, "overrides": [] }, - "id": 6 + "id": 9 }, { "title": "Per-Job-Type Execution Time (p99)", @@ -347,7 +394,7 @@ "h": 8, "w": 24, "x": 0, - "y": 40 + "y": 39 }, "options": { "tooltip": { @@ -376,12 +423,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -389,7 +437,20 @@ }, "overrides": [] }, - "id": 7 + "id": 10 + }, + { + "title": "Overflow", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "collapsed": false, + "panels": [], + "id": 11 }, { "title": "Transaction Overflow Rate", @@ -397,7 +458,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, "y": 48 }, @@ -439,12 +500,13 @@ "custom": { "axisLabel": "Overflows / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -452,7 +514,7 @@ }, "overrides": [] }, - "id": 8 + "id": 12 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 8f7aedfbd5..2a10be92a0 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -220,7 +220,7 @@ "type": "text", "title": "Read This First — Debug Log Requirement", "gridPos": { - "h": 10, + "h": 4, "w": 24, "x": 0, "y": 0 @@ -244,24 +244,24 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 4 }, "panels": [], "id": 2 }, { "type": "stat", - "title": "Nodes By Error Volume", + "title": "Nodes By Error Volume [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most errors.*\n\n###### How it's computed:\n*Count of ERR and FTL log lines per node over the dashboard window, ranked highest first.*\n\n###### Reading it:\n*The top entry is the node in the most trouble. Compare nodes rather than reading an absolute value.*\n\n###### Healthy range:\n*Zero, or a small flat count. ERR is not routine.*\n\n###### Watch for:\n*Any node pulling far ahead of its peers, which usually means a fault local to that node rather than a network condition.*\n\n###### Keywords:\n- **ERR / FTL** *(per line)* — the two most severe xrpld log levels; both survive the default Info threshold.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -318,21 +318,24 @@ "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 3 + "id": 3, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Attack-Like Input", + "title": "Nodes By Attack-Like Input [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are receiving the most malformed or unwanted peer payloads.*\n\n###### How it's computed:\n*Count of Resource fee charges whose reason is useless data, unwanted data, or init drop, per node.*\n\n###### Reading it:\n*These three reasons indicate a peer sent something the node could not use, so a high count is the closest log-derived proxy for abusive input.*\n\n###### Healthy range:\n*Low and flat. Some useless data is normal on a busy overlay.*\n\n###### Watch for:\n*A sharp rise on one node, especially paired with a single dominant IP in the charged-peers table below.*\n\n###### Keywords:\n- **Useless data** *(per charge)* — payload the node could not use, e.g. an empty or malformed message.\n- **Init drop** *(per charge)* — a connection dropped during initialisation.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -389,21 +392,24 @@ "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 4 + "id": 4, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Total Fee Charged", + "title": "Nodes By Total Fee Charged [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are absorbing the most peer-imposed load, weighted by fee amount.*\n\n###### How it's computed:\n*Sum of every resource fee amount parsed from the Resource partition, per node.*\n\n###### Reading it:\n*Weighted by cost rather than event count, so one heavy request at 2000 outranks eight moderate ones at 250.*\n\n###### Healthy range:\n*Proportional to peer count and request volume; compare nodes of similar role.*\n\n###### Watch for:\n*One node far above peers of the same role, which means it is carrying disproportionate peer load.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount: 2000 heavy, 250 moderate, 150 useless data.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -460,21 +466,24 @@ "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 5 + "id": 5, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Manifest Rejection", + "title": "Nodes By Manifest Rejection [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are rejecting the most inbound manifests.*\n\n###### How it's computed:\n*Count of ManifestCache lines with a Stale, Invalid, Revoked, or UntrustedCapacity outcome, per node.*\n\n###### Reading it:\n*A manifest flood shows up here first. Stale dominates normally because peers re-gossip manifests the node already holds.*\n\n###### Healthy range:\n*Workload-dependent; nodes on the same network should sit close together.*\n\n###### Watch for:\n*A single node far ahead, or any Invalid at all, which means a signature failed verification.*\n\n###### Keywords:\n- **Rejection** *(per manifest)* — any non-accepted disposition.\n- **UntrustedCapacity** *(per manifest)* — rate-limit rejection for an unlisted key; rc5+ builds only.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest`\n\n###### Note:\n*Requires debug logs on the ManifestCache partition.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -531,21 +540,24 @@ "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 6 + "id": 6, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Consensus Problem", + "title": "Nodes By Consensus Problem [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most consensus warnings and errors.*\n\n###### How it's computed:\n*Count of LedgerConsensus lines at WRN severity or above, per node.*\n\n###### Reading it:\n*Consensus is a network-wide process, so a single node standing out points at that node rather than the network.*\n\n###### Healthy range:\n*Low. Some warnings occur during normal round churn.*\n\n###### Watch for:\n*One node far above its peers, or a step change after a deploy.*\n\n###### Keywords:\n- **Consensus round** *(per ledger)* — one Open to Establish to Accepted cycle.\n- **Dispute** *(per transaction)* — a transaction peers disagree about including.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::timerEntry / checkConsensus`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -602,21 +614,24 @@ "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 7 + "id": 7, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Job Latency Breach", + "title": "Nodes By Job Latency Breach [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are missing job latency targets most often.*\n\n###### How it's computed:\n*Count of LoadMonitor slow-job lines per node; the emitter only fires above a 500ms threshold.*\n\n###### Reading it:\n*A direct read on which node is most overloaded. Works at the default log level.*\n\n###### Healthy range:\n*Low. A busy node breaches occasionally.*\n\n###### Watch for:\n*A node far ahead of its peers, which usually means disk or CPU pressure local to it.*\n\n###### Keywords:\n- **Breach** *(per job)* — one job execution over the 500ms LoadMonitor threshold.\n\n###### Computation boundary:\n*Result: Per node — a count of BREACHES, never of total job executions.*\n*Derived in the Grafana query. Use the native `job_*` metrics for totals.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts breaches above 500ms only, so it is an exception count and not a latency measure.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -673,21 +688,24 @@ "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 8 + "id": 8, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Sync Instability", + "title": "Nodes By Sync Instability [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are spending the most time out of the full state.*\n\n###### How it's computed:\n*Count of STATE-> transitions into any non-full state, per node.*\n\n###### Reading it:\n*A stable node holds full and emits nothing here, so any non-zero value means it left full that many times.*\n\n###### Healthy range:\n*Zero on a synced node.*\n\n###### Watch for:\n*A repeating count, which means the node is flapping rather than having had one bad moment.*\n\n###### Keywords:\n- **Operating mode** *(per node)* — Disconnected, Connected, Syncing, Tracking, Full.\n- **Flapping** *(per node)* — repeated departures from full.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -744,21 +762,24 @@ "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 9 + "id": 9, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Duplicate Ledger Fetches", + "title": "Nodes By Duplicate Ledger Fetches [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes wasted the most ledger-fetch bandwidth, counted as duplicate SHAMap nodes received.*\n\n###### How it's computed:\n*Sum of the `dupe` counter from `Ledger AS/TX node stats` log lines, per node, over the dashboard window. `dupe` is matched by its own regex because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*Every duplicate is a ledger node the peer sent that this node already held: bytes crossed the network, the peer spent resource credit serving it, and it was discarded.*\n\n###### Healthy range:\n*Workload-dependent. Compare nodes of the same role rather than reading an absolute value; some duplication is unavoidable when fetching one ledger from several peers.*\n\n###### Watch for:\n*A node far ahead of its siblings, which points at that node's fetch logic or peer set rather than a network condition.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a COUNT of duplicates, not a ratio.*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over raw log text.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### Note:\n*Requires debug logs on the InboundLedger partition. For the duplicate RATIO see \"Ledger Node Fetch Duplicate Ratio\" in the Ledger Acquisition Efficiency row — a stat panel cannot render a division, because Loki does not name binary-operation frames and Grafana falls back to \"Value #A\".*\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -815,7 +836,10 @@ "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 10 + "id": 10, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -825,7 +849,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 21 }, "panels": [], "id": 11 @@ -839,10 +863,10 @@ "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -851,13 +875,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -892,17 +917,17 @@ }, { "type": "state-timeline", - "title": "Node State Timeline", + "title": "Node State Timeline [$xrpl_network_type]", "description": "###### What this is:\n*The node's operating state over time, as a discrete timeline.*\n\n###### How it's computed:\n*The most recent `STATE->` transition in each interval, rendered as a state band.*\n\n###### Reading it:\n*Shows at a glance how long the node spent in each state and exactly when it left full.*\n\n###### Healthy range:\n*One unbroken full band across the window.*\n\n###### Watch for:\n*Any band that is not full, and repeated narrow bands, which indicate state flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — one band per server per state.*\n*NOT recorded as a metric anywhere. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### Note:\n*Only transitions are logged, so a node that never changes state produces no data here. Read with the transition-rate panel beside it.*\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -945,7 +970,10 @@ "expr": "sum by (state, service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `STATE->(?P\\w+)` | state != `` | label_format state=`{{if eq .state \"full\"}}Full{{else if eq .state \"syncing\"}}Syncing{{else if eq .state \"tracking\"}}Tracking{{else if eq .state \"connected\"}}Connected{{else if eq .state \"disconnected\"}}Disconnected{{else}}{{.state}}{{end}}` [$__auto]))" } ], - "id": 13 + "id": 13, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -955,7 +983,7 @@ "h": 1, "w": 24, "x": 0, - "y": 56 + "y": 30 }, "panels": [], "id": 14 @@ -965,10 +993,10 @@ "title": "Log Line Rate By Severity", "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's filelog receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -976,13 +1004,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1024,10 +1053,10 @@ "title": "Log Line Rate By Partition (Top $topn)", "description": "###### What this is:\n*The busiest xrpld log partitions by line rate.*\n\n###### How it's computed:\n*Per-second count of log lines grouped by the partition field, limited to the top N series.*\n\n###### Reading it:\n*Shows which subsystem dominates log volume, which is the main cost driver for Loki ingest.*\n\n###### Healthy range:\n*Workload-dependent. Resource, JobQueue, and LedgerConsensus are normally the loudest.*\n\n###### Watch for:\n*A partition suddenly dominating, which usually means a subsystem entered a retry or error loop.*\n\n###### Keywords:\n- **Partition** *(per line)* — the xrpld subsystem that emitted the line, e.g. ManifestCache, Resource, LedgerConsensus.\n\n###### Computation boundary:\n*Result: Per node per partition — a count of log lines.*\n*Derived in the Grafana query; truncated to the top N by Loki's series limit.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -1035,13 +1064,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1086,7 +1116,7 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 39 }, "panels": [], "id": 17 @@ -1096,10 +1126,10 @@ "title": "Manifest Disposition Rate", "description": "###### What this is:\n*Rate of manifest apply outcomes: accepted, stale, revoked, invalid, or rate-limited.*\n\n###### How it's computed:\n*Per-second count of ManifestCache log lines, with the action parsed out of the `Manifest: ;Pk: ...` text.*\n\n###### Reading it:\n*AcceptedNew and AcceptedUpdate are useful work; Stale and UntrustedCapacity are rejections and normally dominate.*\n\n###### Healthy range:\n*Accepted rates are low and bursty. A high sustained Stale rate is normal — peers gossip manifests this node already holds.*\n\n###### Watch for:\n*Any Invalid, which means a signature failed verification, and a sustained UntrustedCapacity climb, which indicates a manifest flood from unlisted keys.*\n\n###### Keywords:\n- **Manifest** *(per validator)* — a signed record binding a validator's master key to its current signing key.\n- **Disposition** *(per manifest)* — the apply outcome: AcceptedNew, AcceptedUpdate, Stale, Revoked, Invalid, UntrustedCapacity.\n- **Stale** *(per manifest)* — sequence number not greater than the one already held; the common benign rejection.\n\n###### Computation boundary:\n*Result: Per node per action — counts log EVENTS, not distinct manifests.*\n*NOT recorded as a metric. `applyManifest` has no instrumentation; this is regex over `logMftAct` output in the Grafana query.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { @@ -1107,13 +1137,14 @@ "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1155,23 +1186,24 @@ "title": "Manifest Accept Vs Reject Rate", "description": "###### What this is:\n*Manifest outcomes collapsed into two series: accepted versus rejected.*\n\n###### How it's computed:\n*Per-second counts of ManifestCache lines, with AcceptedNew and AcceptedUpdate summed as accepted and all other actions summed as rejected.*\n\n###### Reading it:\n*Gives the useful-work fraction of inbound manifest processing without per-action detail.*\n\n###### Healthy range:\n*Rejected normally far exceeds accepted; peers routinely re-gossip known manifests.*\n\n###### Watch for:\n*A rejected rate in the hundreds per second, which indicates a manifest flood consuming the JtManifest job queue.*\n\n###### Keywords:\n- **Accepted** *(per manifest)* — AcceptedNew or AcceptedUpdate; the manifest changed cache state.\n- **Rejected** *(per manifest)* — Stale, Invalid, Revoked, or UntrustedCapacity.\n\n###### Computation boundary:\n*Result: Per node — two summed series over the same log lines.*\n*Derived in the Grafana query. Accepted may legitimately be absent: measured 1,360 AcceptedNew per 7d, so short windows often contain none. Both series carry `or vector(0)` so a zero-accept window renders a flat zero line rather than disappearing.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1220,13 +1252,13 @@ }, { "type": "table", - "title": "Top $topn Manifest Producers By Master Key", + "title": "Top $topn Manifest Producers By Master Key [$xrpl_network_type]", "description": "###### What this is:\n*The master keys responsible for the most manifest log events in the selected window.*\n\n###### How it's computed:\n*Count of ManifestCache lines grouped by the base58 master key parsed from the log text, limited to the top N.\nCounts are per log EVENT, not per distinct manifest.*\n\n###### Reading it:\n*Identifies which validator keys generate the most manifest churn on this node.*\n\n###### Healthy range:\n*A small number of keys with modest repeat counts. Live observation shows roughly 19 to 24 events per key per hour.*\n\n###### Watch for:\n*A single unlisted key with a very high count, which is the signature of a targeted manifest flood.*\n\n###### Keywords:\n- **Master key** *(per validator)* — the long-lived base58 key a manifest is signed under.\n- **Log event** *(per line)* — one manifest apply attempt; the same key recurs as peers re-gossip.\n\n###### Computation boundary:\n*Result: Per master key per node — counts log EVENTS, NOT distinct manifests and NOT distinct keys.*\n*Derived in the Grafana query. Truncated to top N: Loki rejects queries returning over 2000 series, so a true distinct-key count is impossible here.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`logMftAct`\n\n###### Note:\n*Truncated to the top N. Loki rejects a query returning over 2000 series, so a full distinct-key count is not possible here.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 78 + "y": 48 }, "fieldConfig": { "defaults": { @@ -1353,7 +1385,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1363,7 +1398,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 60 }, "panels": [], "id": 21 @@ -1373,10 +1408,10 @@ "title": "Fee Charge Rate By Reason", "description": "###### What this is:\n*Rate at which this node charges peers a resource fee, split by the reason for the charge.*\n\n###### How it's computed:\n*Per-second count of Resource log lines, with the reason parsed from the `for ($)` text.*\n\n###### Reading it:\n*Shows what kind of peer behaviour is costing this node the most. Moderate peer request normally dominates.*\n\n###### Healthy range:\n*Workload-dependent and roughly proportional to peer count and request volume.*\n\n###### Watch for:\n*A rising useless data rate, which means peers are sending malformed or unwanted payloads.*\n\n###### Keywords:\n- **Resource fee** *(per peer)* — an internal load credit charged against a peer, unrelated to XRP transaction fees.\n- **Reason** *(per charge)* — why the charge was applied, e.g. moderate peer request, useless data.\n\n###### Computation boundary:\n*Result: Per node per reason — a count of charge events.*\n*NOT recorded as a metric. Derived in the Grafana query from `Logic::charge` log output.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1384,13 +1419,14 @@ "unit": "suffix: charges/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Charges / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1432,10 +1468,10 @@ "title": "Fee-Weighted Charge Load", "description": "###### What this is:\n*Resource fee charges weighted by the fee amount, rather than counted equally.*\n\n###### How it's computed:\n*Per-second sum of the fee value parsed from each Resource charge line, using LogQL unwrap.*\n\n###### Reading it:\n*A single heavy peer request at 2000 outweighs eight moderate ones at 250, so this ranks real cost rather than event count.*\n\n###### Healthy range:\n*Workload-dependent; should track peer request volume smoothly.*\n\n###### Watch for:\n*Spikes that do not appear in the charge-count panel, which mean a shift toward expensive request types.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount, e.g. 250 moderate, 2000 heavy; higher means costlier.\n\n###### Computation boundary:\n*Result: Per node per reason — a SUM of fee amounts, not a count of events.*\n*Derived in the Grafana query via LogQL `unwrap` over the parsed fee value.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1443,13 +1479,14 @@ "unit": "suffix: fee/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Fee Units / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1488,13 +1525,13 @@ }, { "type": "table", - "title": "Top $topn Charged Peers By IP And Public Key", + "title": "Top $topn Charged Peers By IP And Public Key [$xrpl_network_type]", "description": "###### What this is:\n*The peers accumulating the most resource fee charges, identified by IP address and node public key.*\n\n###### How it's computed:\n*Sum of fee amounts parsed from Resource charge lines, grouped by IP and public key, limited to the top N.*\n\n###### Reading it:\n*This is the direct answer to which peer is costing this node the most, and is the primary abuse-attribution view.*\n\n###### Healthy range:\n*Charges spread across many peers with no single dominant entry.*\n\n###### Watch for:\n*One IP far above the rest, especially paired with the useless data reason, which indicates an abusive or misbehaving peer.*\n\n###### Keywords:\n- **Node public key** *(per peer)* — the peer's base58 identity, stable across reconnects.\n- **IP address** *(per peer)* — source address; the optional `:port` suffix is stripped so one peer is one row.\n\n###### Computation boundary:\n*Result: Per peer per node — a SUM of fee amounts over the dashboard window.*\n*Derived in the Grafana query. Truncated to top N by Loki's 2000-series limit, so this is the head of the distribution, never a total.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Truncated to the top N because of Loki's 2000-series query limit.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 103 + "y": 69 }, "fieldConfig": { "defaults": { @@ -1633,7 +1670,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1643,7 +1683,7 @@ "h": 1, "w": 24, "x": 0, - "y": 117 + "y": 81 }, "panels": [], "id": 25 @@ -1653,10 +1693,10 @@ "title": "Ledger Node Fetch Duplicate Ratio", "description": "###### What this is:\n*The fraction of fetched ledger nodes that were duplicates this node already held.*\n\n###### How it's computed:\n*Sum of the dupe counter divided by the sum of good plus dupe, parsed from the `Ledger AS/TX node stats` log lines. Each field is matched independently because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*This is wasted ledger-fetch bandwidth. A ratio of 0.5 means half of everything fetched was already present.*\n\n###### Healthy range:\n*Below roughly 0.3. Some duplication is unavoidable when fetching from several peers at once.*\n\n###### Watch for:\n*Sustained values above 0.8. Live observation has recorded good:142 dupe:891, an 86 percent duplicate rate, which wastes both bandwidth and peer resource credit.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a ratio of two summed counters, dupe / (good + dupe).*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over the counters in the InboundLedger stats line.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer / takeAsRootNode`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { @@ -1665,13 +1705,14 @@ "max": 1, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duplicate Ratio", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1731,23 +1772,24 @@ "title": "Ledger Node Fetch Rate — Good Vs Duplicate Vs Timeout", "description": "###### What this is:\n*Absolute rate of useful ledger nodes fetched, duplicates received, and acquire timeouts.*\n\n###### How it's computed:\n*Per-second sums of the good, dupe, and timeouts counters parsed from InboundLedger acquire log lines.*\n\n###### Reading it:\n*Gives the absolute volumes behind the duplicate-ratio panel, so a high ratio at trivial volume can be told apart from a high ratio at scale.*\n\n###### Healthy range:\n*Good should exceed duplicate during active sync. Timeouts should stay near zero.*\n\n###### Watch for:\n*A rising timeout series, which means peers are not answering ledger requests and sync will stall.*\n\n###### Keywords:\n- **Good** *(per fetch)* — a useful, previously unheld ledger node.\n- **Timeout** *(per acquire)* — a ledger request a peer never answered.\n\n###### Computation boundary:\n*Result: Per node — SUMS of the parsed counters, giving absolute volumes behind the ratio panel.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { "unit": "suffix: nodes/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Ledger Nodes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1811,7 +1853,7 @@ "h": 1, "w": 24, "x": 0, - "y": 128 + "y": 90 }, "panels": [], "id": 28 @@ -1821,10 +1863,10 @@ "title": "Peer Disconnect Rate By Reason", "description": "###### What this is:\n*Rate of peer connection endings, split by the reason recorded in the log.*\n\n###### How it's computed:\n*Per-second count of Peer log lines matching Timeout, Closed, or a refused connection attempt.*\n\n###### Reading it:\n*Distinguishes clean teardown from failure. Closed is a normal ending; Timeout and Connection refused are not.*\n\n###### Healthy range:\n*Closed dominant with a low, steady background of the others.*\n\n###### Watch for:\n*A Timeout rate approaching the Closed rate, which points at network trouble or unresponsive peers.*\n\n###### Keywords:\n- **Closed** *(per peer)* — a clean connection teardown; the normal ending.\n- **Timeout** *(per peer)* — the peer stopped responding.\n- **Connection refused** *(per attempt)* — an outbound attempt the remote rejected.\n\n###### Computation boundary:\n*Result: Per node per outcome — a count of peer lifecycle events.*\n*Derived in the Grafana query. Note `overlay_peer_disconnects` exists as a metric but carries no reason breakdown, which is what this panel adds.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close / onTimer`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { @@ -1832,13 +1874,14 @@ "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1880,23 +1923,24 @@ "title": "Peer Handshake And Accept Rate", "description": "###### What this is:\n*Rate of completed peer handshakes and accepted inbound connections.*\n\n###### How it's computed:\n*Per-second count of PeerFinder log lines matching handshake and accept events.*\n\n###### Reading it:\n*Read together with the disconnect panel: healthy churn has handshakes roughly balancing disconnects.*\n\n###### Healthy range:\n*Low and steady once the node has a full peer slate.*\n\n###### Watch for:\n*A high handshake rate alongside a high disconnect rate, which means peers connect and immediately drop.*\n\n###### Keywords:\n- **Handshake** *(per peer)* — protocol negotiation completed with a peer.\n- **Accept** *(per peer)* — an inbound connection admitted to a peer slot.\n\n###### Computation boundary:\n*Result: Per node — counts of PeerFinder events.*\n*Derived in the Grafana query.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/Logic.h)\n\n###### Function:\n`Logic::on_handshake / on_accept`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1951,7 +1995,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 99 }, "panels": [], "id": 31 @@ -1961,10 +2005,10 @@ "title": "Consensus Phase Transition Rate", "description": "###### What this is:\n*Rate of consensus phase transitions into Open, Establish, and Accepted.*\n\n###### How it's computed:\n*Per-second count of log lines announcing a ConsensusPhase transition, grouped by the target phase.*\n\n###### Reading it:\n*A healthy node cycles Open to Establish to Accepted once per ledger, so all three series should track together.*\n\n###### Healthy range:\n*Roughly one transition per phase per ledger interval, near 0.25 per second on a 4-second close.*\n\n###### Watch for:\n*Establish transitions outnumbering Accepted, which means rounds start but fail to converge.*\n\n###### Keywords:\n- **Consensus phase** *(per round)* — Open collects transactions, Establish converges on a set, Accepted applies it.\n- **Transition** *(per round)* — one phase change; a healthy node cycles all three once per ledger.\n\n###### Computation boundary:\n*Result: Per node per phase — a count of transitions.*\n*Derived in the Grafana query by regex over the phase-change log line.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::phase transition logging`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -1972,13 +2016,14 @@ "unit": "suffix: transitions/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2020,10 +2065,10 @@ "title": "Consensus Operating Mode Rate", "description": "###### What this is:\n*Rate of consensus heartbeat log lines, split by the operating mode reported in each.*\n\n###### How it's computed:\n*Per-second count of ConsensusLogger heartbeat lines, with the mode parsed from the `mode: ` text.*\n\n###### Reading it:\n*This acts as a mode-occupancy proxy: the heartbeat fires about once per second, so the dominant series is the node's current mode.*\n\n###### Healthy range:\n*Almost entirely the full series on a synced node.*\n\n###### Watch for:\n*Any sustained syncing, observing, or tracking share, which means the node is not participating normally.*\n\n###### Keywords:\n- **Consensus mode** *(per node)* — the node's participation level: full proposing, observing, syncing, tracking.\n- **Heartbeat** *(per second)* — the ConsensusLogger timer line, emitted roughly once per second.\n\n###### Computation boundary:\n*Result: Per node per mode — a count of heartbeat lines, used as a mode-occupancy PROXY rather than a true duration.*\n*Derived in the Grafana query. For exact durations use the state_accounting metrics on the Node Health dashboard.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`ConsensusLogger heartbeat`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -2031,13 +2076,14 @@ "unit": "suffix: beats/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Heartbeats / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2079,10 +2125,10 @@ "title": "Validator List Quorum And Trusted Set Size", "description": "###### What this is:\n*The quorum threshold and trusted validator count this node computed, as recorded in the log.*\n\n###### How it's computed:\n*Quorum and total are parsed from the `Using quorum of N for new set of M trusted validators` line and plotted as values.*\n\n###### Reading it:\n*Confirms the node agrees with the network on how many validators it trusts and how many must agree.*\n\n###### Healthy range:\n*Stable, with quorum at roughly 80 percent of the trusted total.*\n\n###### Watch for:\n*A drop in the trusted total, which shrinks the quorum and weakens the node's safety margin.*\n\n###### Keywords:\n- **Quorum** *(per node)* — how many trusted validators must agree; normally about 80 percent of the trusted set.\n- **Trusted set** *(per node)* — validators from the UNL this node currently trusts.\n\n###### Computation boundary:\n*Result: Per node — the last VALUE parsed from the log line, not a rate or count.*\n*Derived in the Grafana query via `unwrap`. Overlaps the `unl_quorum` metric; prefer that metric for the value and this panel for churn events.*\n\n###### Source:\n[ValidatorList.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorList.cpp)\n\n###### Function:\n`ValidatorList::updateTrusted`\n\n###### References:\n[Unique Node List](https://xrpl.org/docs/concepts/consensus-protocol/unique-node-list)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 150 + "y": 108 }, "fieldConfig": { "defaults": { @@ -2090,17 +2136,19 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, + "lineWidth": 1, "fillOpacity": 0, "axisLabel": "Validators", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] }, @@ -2151,7 +2199,7 @@ "h": 1, "w": 24, "x": 0, - "y": 160 + "y": 116 }, "panels": [], "id": 35 @@ -2161,10 +2209,10 @@ "title": "Slow Job Run Time (p99)", "description": "###### What this is:\n*The 99th percentile run time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the run millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Identifies which job types are the slowest when they do run long.*\n\n###### Healthy range:\n*Only a handful of job names should appear at all. Values in the tens of seconds indicate real stalls.*\n\n###### Watch for:\n*ProcessLData or InboundLedger dominating, which is the signature of the cold-read sync bottleneck.*\n\n###### Keywords:\n- **Job** *(per task)* — a unit of work on xrpld's job queue, e.g. ProcessLData, InboundLedger.\n- **Run time** *(per job)* — time spent executing, excluding queue wait.\n- **Breach** *(per job)* — an execution exceeding the 500ms LoadMonitor threshold; only these are logged.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY, never over all executions.*\n*Derived in the Grafana query via `unwrap`. For total job counts and latencies use the native `job_*` metrics on the Job Queue Analysis dashboard.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*These lines are only emitted above a 500ms latency threshold, so this panel counts BREACHES, never total job executions. Use the Job Queue Analysis dashboard for totals.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2173,13 +2221,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Run Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2221,10 +2270,10 @@ "title": "Slow Job Breach Rate By Job Name (Top $topn)", "description": "###### What this is:\n*How often each job type breaches its latency target.*\n\n###### How it's computed:\n*Per-second count of LoadMonitor job lines grouped by job name, limited to the top N.*\n\n###### Reading it:\n*Read with the p99 panel: a job can breach rarely but severely, or often but mildly.*\n\n###### Healthy range:\n*Near zero. Any sustained non-zero rate means a job type is routinely missing its target.*\n\n###### Watch for:\n*A step change after a deploy, which points at a regression in that job's path.*\n\n###### Keywords:\n- **Breach** *(per job)* — an execution over the 500ms threshold.\n- **Job type** *(per task)* — the named job class, e.g. sweep, ProcessLData.\n\n###### Computation boundary:\n*Result: Per node per job type — a count of BREACHES, not of executions.*\n*Derived in the Grafana query; truncated to top N.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2232,13 +2281,14 @@ "unit": "suffix: breaches/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Breaches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2280,10 +2330,10 @@ "title": "Slow Job Queue Wait Time (p99)", "description": "###### What this is:\n*The 99th percentile queue wait time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the wait millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Separates queueing delay from execution cost. High wait with low run means the job queue is saturated rather than the work being slow.*\n\n###### Healthy range:\n*Near zero. Live observation shows wait at 0ms for most breaches, meaning run time is the cause.*\n\n###### Watch for:\n*Wait time rising above run time, which shifts the diagnosis from slow work to a saturated job queue.*\n\n###### Keywords:\n- **Wait time** *(per job)* — time queued before execution began.\n- **Run versus wait** *(per job)* — high wait means a saturated queue; high run means slow work.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 171 + "y": 125 }, "fieldConfig": { "defaults": { @@ -2292,13 +2342,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Queue Wait Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2343,7 +2394,7 @@ "h": 1, "w": 24, "x": 0, - "y": 181 + "y": 133 }, "panels": [], "id": 39 @@ -2353,10 +2404,10 @@ "title": "Warning And Error Rate By Partition (Top $topn)", "description": "###### What this is:\n*Rate of WRN, ERR, and FTL log lines by partition.*\n\n###### How it's computed:\n*Per-second count of log lines at warning severity or above, grouped by partition and limited to the top N.*\n\n###### Reading it:\n*This is the one row that works fully at default log level, so it is the first place to look on an unmodified node.*\n\n###### Healthy range:\n*Low and flat. LoadMonitor warnings are expected on a busy node.*\n\n###### Watch for:\n*Any ERR or FTL series appearing, and step changes in a partition that is normally quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — WRN, ERR, or FTL; the levels a default-configured node still writes.\n- **Partition** *(per line)* — the emitting xrpld subsystem.\n\n###### Computation boundary:\n*Result: Per node per partition per severity — a count of log lines.*\n*Derived in the Grafana query. This row is the only one that works fully at default log level.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 182 + "y": 134 }, "fieldConfig": { "defaults": { @@ -2364,13 +2415,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2431,10 +2483,10 @@ "title": "Recent Warnings And Errors", "description": "###### What this is:\n*The most recent log lines at warning severity or above.*\n\n###### How it's computed:\n*Raw log stream filtered to WRN, ERR, and FTL, newest first.*\n\n###### Reading it:\n*Use this to read the actual text behind a spike in the rate panel beside it.*\n\n###### Healthy range:\n*Sparse. LoadMonitor slow-job warnings are the common benign entry.*\n\n###### Watch for:\n*Repeating identical errors, which indicate a stuck retry loop rather than a transient fault.*\n\n###### Keywords:\n- **Log stream** *(per node)* — the raw lines behind the aggregate rates, newest first.\n\n###### Computation boundary:\n*Result: Raw log lines, no aggregation.*\n*Rendered directly from Loki; no computation applied.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 182 + "h": 12, + "w": 24, + "x": 0, + "y": 142 }, "fieldConfig": { "defaults": {}, diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index e17bb2dda4..2da2cfbd12 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -39,15 +39,28 @@ "id": null, "links": [], "panels": [ + { + "title": "Peer Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "P90 Peer Latency", "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -72,12 +85,13 @@ "custom": { "axisLabel": "Latency (ms)", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -100,17 +114,31 @@ } }, "overrides": [] - } + }, + "id": 2 }, { - "title": "Insane/Diverged Peers", + "title": "Peer Health", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 3 + }, + { + "title": "Insane/Diverged Peers [$xrpl_network_type]", "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Keywords:\n- **Insane / diverged peers** *(per node)* \u2014 connected peers whose ledger state disagrees with the network \u2014 possibly on a fork or misbehaving.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -148,20 +176,25 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Higher Version Peers %", + "title": "Higher Version Peers % [$xrpl_network_type]", "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 16 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -204,17 +237,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { - "title": "Upgrade Recommended", + "title": "Upgrade Recommended [$xrpl_network_type]", "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 24 + "y": 14 }, "options": { "tooltip": { @@ -271,7 +308,24 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 6 + }, + { + "title": "Disconnects & Connection Mix", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Resource Disconnects", @@ -279,9 +333,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 32 + "y": 19 }, "options": { "tooltip": { @@ -306,29 +360,32 @@ "custom": { "axisLabel": "Disconnects", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Inbound vs Outbound Peers", + "title": "Inbound vs Outbound Peers [$xrpl_network_type]", "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 40 + "y": 27 }, "options": { "orientation": "horizontal", @@ -367,7 +424,8 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [ { @@ -401,7 +459,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 4e32e7e606..3a0cde97bc 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -49,15 +49,16 @@ "y": 0 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 1 }, { - "title": "Agreement % (1h)", + "title": "Agreement % (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last hour where this validator agreed with the network consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 1-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; green at 95%+, yellow from 80%, red below.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*Values below 80%, meaning the validator frequently disagrees with consensus.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, "y": 1 }, @@ -102,17 +103,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 }, { - "title": "Agreement % (24h)", + "title": "Agreement % (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last 24 hours where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 24-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; a smoother, longer-term view than the 1h stat.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A sustained dip below 90%, which can indicate configuration drift or a network partition.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 9 + "h": 4, + "w": 12, + "x": 12, + "y": 1 }, "options": { "tooltip": { @@ -155,17 +160,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 }, { - "title": "Agreements vs Missed (1h)", + "title": "Agreements vs Missed (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last hour.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 1-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate; missed should be small.*\n\n###### Healthy range:\n*Missed near 0.*\n\n###### Watch for:\n*A high missed count, meaning the validator is skipping consensus rounds.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 17 + "y": 5 }, "options": { "orientation": "horizontal", @@ -223,17 +232,21 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Agreements vs Missed (24h)", + "title": "Agreements vs Missed (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last 24 hours.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 24-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate over the full day.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A growing missed share, signalling longer-term reliability problems.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 + "h": 4, + "w": 12, + "x": 12, + "y": 5 }, "options": { "orientation": "horizontal", @@ -291,7 +304,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { "title": "Validation Rates", @@ -300,20 +317,21 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 9 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 6 }, { - "title": "Validation Rate", + "title": "Validation Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations this node sends per minute.*\n\n###### How it's computed:\n*Per-second rate of the sent-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should track the ledger close cadence; roughly one validation per closed ledger.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*A drop toward zero, meaning the validator has stopped participating.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsSent (caller RCLConsensus.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 34 + "y": 10 }, "options": { "tooltip": { @@ -354,17 +372,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "Validations Checked Rate", + "title": "Validations Checked Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations received from peers and checked per minute.*\n\n###### How it's computed:\n*Per-second rate of the checked-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Reflects how much validation traffic the network is delivering to this node.*\n\n###### Healthy range:\n*Workload-dependent; scales with trusted validator count.*\n\n###### Watch for:\n*A sudden collapse, which suggests peer connectivity loss or network isolation.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsChecked (caller NetworkOPs.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -389,17 +411,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 }, { - "title": "Amendment Blocked", + "title": "Amendment Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node is amendment-blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the amendment-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means an enabled amendment is unsupported by this build.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which halts validation and requires a software upgrade.*\n\n###### Keywords:\n- **Amendment blocked** *(per node)* \u2014 the node has halted because the network enabled an amendment its software version does not support.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Amendment blocked](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 50 + "y": 14 }, "options": { "tooltip": { @@ -456,17 +482,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 }, { - "title": "UNL Expiry (days)", + "title": "UNL Expiry (days) [$xrpl_network_type]", "description": "###### What this is:\n*Days remaining until the current UNL (trusted validator list) expires.*\n\n###### How it's computed:\n*Instantaneous gauge reading of days-to-expiry.*\n\n###### Reading it:\n*Higher is safer; green at 30+, yellow under 7, red at expiry.*\n\n###### Healthy range:\n*30+ days.*\n\n###### Watch for:\n*Fewer than 7 days, after which the node loses its trusted validator set if not renewed.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 58 + "h": 4, + "w": 12, + "x": 12, + "y": 14 }, "options": { "tooltip": { @@ -507,17 +537,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { - "title": "UNL Blocked", + "title": "UNL Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node's UNL is blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the UNL-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means validator trust cannot be established.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which can stop the node participating in consensus.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n- **UNL blocked** *(per node)* \u2014 the node cannot establish a usable trusted validator list, so it cannot safely validate.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [UNL blocked](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 66 + "y": 18 }, "options": { "tooltip": { @@ -574,7 +608,11 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 }, { "title": "Agreement/Missed Counters (Rate)", @@ -582,9 +620,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 74 + "y": 22 }, "options": { "tooltip": { @@ -616,12 +654,13 @@ "custom": { "axisLabel": "Validations / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -644,7 +683,8 @@ ] } ] - } + }, + "id": 12 }, { "title": "Server State & Consensus", @@ -653,20 +693,21 @@ "h": 1, "w": 24, "x": 0, - "y": 82 + "y": 30 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 13 }, { - "title": "Validation Quorum", + "title": "Validation Quorum [$xrpl_network_type]", "description": "###### What this is:\n*Minimum number of trusted validations required to declare a ledger fully validated.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the current quorum requirement.*\n\n###### Reading it:\n*Tracks the quorum derived from the active validator list; changes when the list changes.*\n\n###### Healthy range:\n*Stable at the network-appropriate value.*\n\n###### Watch for:\n*An unexpected drop, which can weaken consensus safety guarantees.*\n\n###### Keywords:\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-quorum)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 83 + "y": 31 }, "options": { "tooltip": { @@ -688,10 +729,15 @@ "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "unit": "none", - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 14 }, { "title": "State Value Timeline", @@ -699,9 +745,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 91 + "y": 35 }, "options": { "tooltip": { @@ -726,29 +772,31 @@ "custom": { "axisLabel": "State", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 15 }, { - "title": "Time in Current State", + "title": "Time in Current State [$xrpl_network_type]", "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 99 + "y": 43 }, "options": { "tooltip": { @@ -773,17 +821,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 16 }, { - "title": "State Changes Rate", + "title": "State Changes Rate [$xrpl_network_type]", "description": "###### What this is:\n*Rate of server operating-state changes per hour.*\n\n###### How it's computed:\n*Per-hour rate of the state-change counter, averaged over a 1-hour window.*\n\n###### Reading it:\n*Near zero is healthy; each increment is one state transition.*\n\n###### Healthy range:\n*Near 0 changes per hour.*\n\n###### Watch for:\n*Frequent transitions, which point to network instability or configuration problems.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementStateChanges (caller NetworkOPs.cpp)`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 107 + "h": 4, + "w": 12, + "x": 12, + "y": 43 }, "options": { "tooltip": { @@ -824,17 +876,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 17 }, { - "title": "Ledgers Closed Rate", + "title": "Ledgers Closed Rate [$xrpl_network_type]", "description": "###### What this is:\n*Ledgers closed per minute by this node.*\n\n###### How it's computed:\n*Per-second rate of the ledgers-closed counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should match the network's steady close cadence.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*Deviation from the expected cadence, which indicates consensus timing trouble or the node falling behind.*\n\n###### Keywords:\n- **Ledgers closed rate** *(per node)* \u2014 how many ledgers this node closed per minute; should match the network close cadence.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgersClosed (caller RCLConsensus.cpp)`\n\n###### References:\n[Ledgers closed rate](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-closed-rate)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 115 + "y": 47 }, "options": { "tooltip": { @@ -875,17 +931,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 18 }, { - "title": "Agreement % (7d)", + "title": "Agreement % (7d) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the trailing 7 days where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 7-day agreement percentage.*\n\n###### Reading it:\n*The long-term reliability window; higher is better.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A gradual decline, which reflects chronic rather than transient disagreement.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 123 + "h": 4, + "w": 12, + "x": 12, + "y": 47 }, "options": { "tooltip": { @@ -928,7 +988,11 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 19 }, { "title": "Agreements vs Missed (7d)", @@ -936,9 +1000,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 131 + "y": 51 }, "options": { "tooltip": { @@ -972,11 +1036,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 20 } ], "schemaVersion": 39,