From b50aa17aee96fa25bb83ad161264b21930f114fd Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:48:26 +0100 Subject: [PATCH 1/3] fix(consensus): include ConsensusSpanNames.h where its symbols are used Both files use telemetry::consensus::span::* but reached the declarations only transitively, which misc-include-cleaner rejects. Add the direct include now that the header lives under xrpl/consensus/. --- src/xrpld/app/consensus/RCLConsensus.cpp | 1 + src/xrpld/overlay/detail/PeerImp.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 508ac2bf14..e68b226d42 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index a6375ace06..b7fd25a68b 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include From 7da5ac5992cc922008c4effc646b12d51decec06 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:20:02 +0100 Subject: [PATCH 2/3] docs(telemetry): correct nudb_bytes and NuDB found-ratio descriptions nudb_bytes was documented as a NuDB file size, one place even claiming a filesystem stat. It observes Database::getStoreSize(), which sums the object payloads this process has written. It excludes NuDB's keys, bucket padding and log, and resets with the process. node_written_bytes calls the same accessor, so the two series are equal by construction and a write-amplification ratio built from them is a constant 1.0. Neither Backend nor Database exposes a file-size accessor, so nothing reports on-disk size today. The Ledger Data & Sync panel plotting node_reads_hit / node_reads_total was titled "NuDB Cache Hit Ratio" and described as reads served from cache. fetchHitCount_ increments whenever a fetch returned an object, whatever served it, so the ratio is a found rate. It reads near 100% while every fetch goes to disk, which made the cold-read failure mode look impossible. Renamed to "NuDB Read Found Ratio" and rewrote the guidance to pair it with read latency. Co-Authored-By: Claude Opus 5 (1M context) --- OpenTelemetryPlan/06-implementation-phases.md | 16 +++++++++++---- OpenTelemetryPlan/Phase7_taskList.md | 20 +++++++++++++------ .../grafana/dashboards/ledger-data-sync.json | 10 +++++----- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index 29b29fff18..52f811818a 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -1289,15 +1289,23 @@ xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external **Task 7.13: Storage Detail Observable Gauge** -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------- | --------------- | ----- | ---------------------------------------- | -| `xrpld_storage_detail` | `nudb_bytes` | int64 | NuDB backend file size (filesystem stat) | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | --------------- | ----- | ---------------------------------------------------- | +| `xrpld_storage_detail` | `nudb_bytes` | int64 | `Database::getStoreSize()` — cumulative object bytes | + +Despite the name, this is not a filesystem measurement. `getStoreSize()` sums the +object payloads this process has written, so it excludes NuDB's keys, bucket padding +and log, and it resets with the process while the files on disk do not. It is the +same accessor `node_written_bytes` uses, so the two series are equal by construction +and any write-amplification ratio built from the pair is a constant 1.0. There is no +file-size accessor on `Backend` or `Database`, so no metric reports the store's +on-disk size today. **File**: `src/xrpld/telemetry/MetricsRegistry.cpp` **Exit Criteria**: -- [ ] NuDB file size reported in bytes +- [ ] Cumulative stored object bytes reported - [ ] Gracefully returns 0 if NuDB not configured --- diff --git a/OpenTelemetryPlan/Phase7_taskList.md b/OpenTelemetryPlan/Phase7_taskList.md index 971372fdb0..d7ce34b7b8 100644 --- a/OpenTelemetryPlan/Phase7_taskList.md +++ b/OpenTelemetryPlan/Phase7_taskList.md @@ -482,20 +482,28 @@ validatorParticipationGauge_ = meter_->CreateInt64ObservableGauge( > **Source**: [External Dashboard Parity](./06-implementation-phases.md#appendix-external-dashboard-parity) -**Objective**: Export NuDB-specific storage size and initial sync duration. +**Objective**: Export NuDB-specific stored-bytes total and initial sync duration. **Gauge label values**: -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------- | ------------------------------- | ------ | ----------------------------- | -| `xrpld_storage_detail` | `nudb_bytes` | int64 | NuDB backend file size | -| `xrpld_sync_info` | `initial_sync_duration_seconds` | double | Time from start to first FULL | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------------------- | ------ | ---------------------------------------------------- | +| `xrpld_storage_detail` | `nudb_bytes` | int64 | `Database::getStoreSize()` — cumulative object bytes | +| `xrpld_sync_info` | `initial_sync_duration_seconds` | double | Time from start to first FULL | + +Despite the name, `nudb_bytes` is not a file size. `getStoreSize()` sums the object +payloads this process has written, so it excludes NuDB's keys, bucket padding and +log, and it resets when the process restarts while the files on disk do not. It is +the same accessor `node_written_bytes` uses, so the two series are equal by +construction and any write-amplification ratio built from the pair is a constant 1.0. +Neither `Backend` nor `Database` exposes a file-size accessor, so no metric reports +the store's on-disk size today. **Key modified files**: `src/xrpld/telemetry/MetricsRegistry.h/.cpp` **Exit Criteria**: -- [ ] NuDB file size reported in bytes (0 if NuDB not configured) +- [ ] Cumulative stored object bytes reported (0 if NuDB not configured) - [ ] Sync duration captured once and remains stable after reaching FULL --- diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 2909d877cd..bfeef82e78 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1079,7 +1079,7 @@ }, { "title": "NuDB Read Latency", - "description": "###### What this is:\n*Average nodestore (NuDB) read latency: how long each on-disk object read takes. The direct disk-layer cost that backs up the job queue.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_duration_us\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]), in us.*\n\n###### Reading it:\n*Low single-digit us when the OS page cache is warm; tens-to-hundreds of us when reads hit the disk. Rises sharply during cold-cache catch-up.*\n\n###### Healthy range:\n*< ~10us/read warm; higher is expected briefly after a wipe/restart.*\n\n###### Watch for:\n*Sustained high us/read together with low cache hit % (below) = disk-bound reads are the bottleneck; check EBS IOPS / io scheduler latency.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`NuDB backend read path`", + "description": "###### What this is:\n*Average nodestore (NuDB) read latency: how long each on-disk object read takes. The direct disk-layer cost that backs up the job queue.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_duration_us\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]), in us.*\n\n###### Reading it:\n*Low single-digit us when the OS page cache is warm; tens-to-hundreds of us when reads hit the disk. Rises sharply during cold-cache catch-up.*\n\n###### Healthy range:\n*< ~10us/read warm; higher is expected briefly after a wipe/restart.*\n\n###### Watch for:\n*Sustained high us/read is the disk-bound signal on its own; the found ratio below stays near 100% even then, so do not wait for it to drop. Check EBS IOPS / io scheduler latency.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`NuDB backend read path`", "type": "timeseries", "gridPos": { "h": 8, @@ -1162,8 +1162,8 @@ "id": 22 }, { - "title": "NuDB Cache Hit Ratio", - "description": "###### What this is:\n*Fraction of nodestore reads served from cache rather than disk. A cold cache (fresh NuDB after a wipe) forces almost every read to disk - the root cause of high read latency during catch-up.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]).*\n\n###### Reading it:\n*Near 1.0 (100%) on a warm node. Near 0 right after a wipe/restart, climbing as the working set warms.*\n\n###### Healthy range:\n*> ~0.9 warm; near 0 briefly after a cold start.*\n\n###### Watch for:\n*Persistently low hit ratio on a node that should be warm = working set does not fit cache, or continuous re-acquisition churning the cache.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`TaggedCache hit accounting`", + "title": "NuDB Read Found Ratio", + "description": "###### What this is:\n*Fraction of nodestore fetches that found the object they asked for. This is not a cache hit ratio: `node_reads_hit` counts every fetch that returned an object, whatever served it, so a fetch that went all the way to disk still counts here.*\n\n###### How it's computed:\n*rate(nodestore_state{metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{metric=\"node_reads_total\"}[$__rate_interval]).*\n\n###### Reading it:\n*Normally sits near 1.0 (100%) on any node that has the data, warm or cold, because a synced node almost always finds what it asks for. It does not fall when the page cache goes cold.*\n\n###### Healthy range:\n*Near 1.0. A ratio well below 1.0 means fetches are missing, which points at a gap in local history rather than at cache pressure.*\n\n###### Watch for:\n*Never read this panel on its own. A ~100% found ratio at over 100 microseconds per read is the cold-read signature, not a healthy cache: the data is found every time and paid for every time. Always pair it with NuDB Read Latency.*\n\n###### Source:\n[nodestore/Database.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/Database.cpp)\n\n###### Function:\n`Database::fetchNodeObject`", "type": "timeseries", "gridPos": { "h": 8, @@ -1184,7 +1184,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Hit Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { @@ -1192,7 +1192,7 @@ "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "unit": "percentunit", "custom": { - "axisLabel": "Hit Ratio", + "axisLabel": "Found Ratio", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", From 8ade2b43d9350f2dbccb98c71f0cd6f7538ca8aa Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:23:41 +0100 Subject: [PATCH 3/3] docs(telemetry): rename node-health found-ratio and stored-bytes panels node_reads_hit is a found count, not a cache hit. fetchHitCount_ increments whenever a fetch returned an object, whatever served it, and a node with online_delete uses DatabaseRotatingImp which has no NodeObject cache at all. The ratio therefore reads near 100% while every fetch goes to disk, which made the cold-read failure mode look impossible on the board. - Phase9_taskList: node_reads_hit is a found count, not cache-served reads. - node-health: "NodeStore Cache Hit Ratio" -> "NodeStore Read Found Ratio", legends "Cache Hit Ratio" -> "Found Ratio" and "Reads Hit (cache)" -> "Reads Found"; corrected the ratio panel's axis label, which read "Operations / Sec" on a percentunit panel. - node-health: "NuDB Storage Size" -> "NuDB Stored Bytes". nudb_bytes observes getStoreSize(), the accumulator node_written_bytes also reads, so it is cumulative object-payload bytes and not a filesystem measurement. - ledger-data-sync: point the read-latency discriminator at the renamed panel. Co-Authored-By: Claude Opus 5 (1M context) --- OpenTelemetryPlan/Phase9_taskList.md | 2 +- .../grafana/dashboards/ledger-data-sync.json | 2 +- .../grafana/dashboards/node-health.json | 20 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/OpenTelemetryPlan/Phase9_taskList.md b/OpenTelemetryPlan/Phase9_taskList.md index e45c5bcd77..a172cd2e52 100644 --- a/OpenTelemetryPlan/Phase9_taskList.md +++ b/OpenTelemetryPlan/Phase9_taskList.md @@ -42,7 +42,7 @@ These metrics serve multiple external consumer categories identified during rese - In `src/libxrpl/nodestore/Database.cpp`, extend existing `beast::insight` registrations to add: - Gauge: `node_reads_total` (cumulative read operations) - - Gauge: `node_reads_hit` (cache-served reads) + - Gauge: `node_reads_hit` (fetches that found an object — not a cache hit; `fetchHitCount_` increments whatever served the fetch) - Gauge: `node_writes` (cumulative write operations) - Gauge: `node_written_bytes` (cumulative bytes written) - Gauge: `node_read_bytes` (cumulative bytes read) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 755d040a85..8d27906799 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1481,7 +1481,7 @@ }, { "title": "NodeStore Read Latency (Bottleneck Discriminator)", - "description": "###### What this is:\n*The single measurement that separates the two ways a ledger sync stalls. Both modes look identical from the job queue -- the ledgerData lane sits at its concurrency limit of 3 with work waiting -- so lane occupancy on its own diagnoses nothing. Read latency does separate them, because the two modes load opposite ends of the storage path.*\n\n###### How it's computed:\n*Two views of the same quantity. Lifetime is the nodestore_state{metric=\"read_mean_us\"} gauge, which is total fetch microseconds divided by total fetches since process start, so it moves slowly and shows the run as a whole. Windowed is rate(node_reads_duration_us) / rate(node_reads_total) over the panel interval, so it reacts within one scrape. The axis is logarithmic because the two modes differ by more than an order of magnitude and a linear axis flattens the lower one.*\n\n###### Reading it:\n*Below about 10 microseconds per read means the backend is answering from the page cache and reads are not the constraint; if the lane is still full, the cost is on the write side -- check NuDB Writer Queue Depth next. Above about 100 microseconds per read, together with a high hit ratio on NuDB Cache Hit Ratio, means objects are being found but paid for with disk latency on every access. High latency with a low hit ratio is a different thing again: the working set does not fit and misses are scanning every backend. The threshold lines at 10 and 100 mark the two boundaries.*\n\n###### Healthy range:\n*Single-digit microseconds per read on a warm synced node, with lifetime and windowed close together.*\n\n###### Watch for:\n*Windowed rising well above lifetime: the recent window is much worse than the run average, which is the earliest reading of a cache that has just gone cold. A measured cold-read episode peaked at 223 microseconds and settled in the 13-35 range while still hitting 88 percent or better, and that shape hung a node for roughly 25 minutes. Also treat a flat 8-9 microsecond line during a stall as informative rather than reassuring -- it rules reads out and points at the write path.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`Database::getFetchDurationUs / NuDB backend read path`", + "description": "###### What this is:\n*The single measurement that separates the two ways a ledger sync stalls. Both modes look identical from the job queue -- the ledgerData lane sits at its concurrency limit of 3 with work waiting -- so lane occupancy on its own diagnoses nothing. Read latency does separate them, because the two modes load opposite ends of the storage path.*\n\n###### How it's computed:\n*Two views of the same quantity. Lifetime is the nodestore_state{metric=\"read_mean_us\"} gauge, which is total fetch microseconds divided by total fetches since process start, so it moves slowly and shows the run as a whole. Windowed is rate(node_reads_duration_us) / rate(node_reads_total) over the panel interval, so it reacts within one scrape. The axis is logarithmic because the two modes differ by more than an order of magnitude and a linear axis flattens the lower one.*\n\n###### Reading it:\n*Below about 10 microseconds per read means the backend is answering from the page cache and reads are not the constraint; if the lane is still full, the cost is on the write side -- check NuDB Writer Queue Depth next. Above about 100 microseconds per read, together with a high found ratio on NuDB Read Found Ratio, means objects are being found but paid for with disk latency on every access. High latency with a low found ratio is a different thing again: the working set does not fit and misses are scanning every backend. The threshold lines at 10 and 100 mark the two boundaries.*\n\n###### Healthy range:\n*Single-digit microseconds per read on a warm synced node, with lifetime and windowed close together.*\n\n###### Watch for:\n*Windowed rising well above lifetime: the recent window is much worse than the run average, which is the earliest reading of a cache that has just gone cold. A measured cold-read episode peaked at 223 microseconds and settled in the 13-35 range while still hitting 88 percent or better, and that shape hung a node for roughly 25 minutes. Also treat a flat 8-9 microsecond line during a stall as informative rather than reassuring -- it rules reads out and points at the write path.*\n\n###### Source:\n[nodestore/backend/NuDBFactory.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/backend/NuDBFactory.cpp)\n\n###### Function:\n`Database::getFetchDurationUs / NuDB backend read path`", "type": "timeseries", "gridPos": { "h": 8, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 2564a7195d..61379e639e 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -485,7 +485,7 @@ }, { "title": "NodeStore Throughput", - "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "description": "###### What this is:\n*Object-store read, found and write operation rates. The found series is `node_reads_hit`, which counts fetches that returned an object whatever served them, so it is not a cache-hit count.*\n\n###### How it's computed:\n*Per-second rates of the read, found and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with found tracking almost all reads on a node that has the data.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { "x": 0, @@ -512,7 +512,7 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]), \"series\", \"Reads Hit (cache)\", \"\", \"\")", + "expr": "label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]), \"series\", \"Reads Found\", \"\", \"\")", "refId": "B" }, { @@ -542,8 +542,8 @@ } }, { - "title": "NodeStore Cache Hit Ratio", - "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "title": "NodeStore Read Found Ratio", + "description": "###### What this is:\n*Fraction of object-store fetches that found the object they asked for. This is not a cache hit ratio: `node_reads_hit` counts every fetch that returned an object, whatever served it, so a fetch that went all the way to disk still counts here.*\n\n###### How it's computed:\n*Rate of node_reads_hit divided by rate of node_reads_total over the panel interval.*\n\n###### Reading it:\n*Normally sits near 1.0 (100%) on any node that has the data, warm or cold. It does not fall when the page cache goes cold.*\n\n###### Healthy range:\n*Near 1.0. A ratio well below 1.0 means fetches are missing, which points at a gap in local history rather than at cache pressure.*\n\n###### Watch for:\n*Never read this panel on its own. A ~100% found ratio at over 100 microseconds per read is the cold-read signature, not a healthy cache: the data is found every time and paid for every time. Pair it with read latency.*\n\n###### Source:\n[nodestore/Database.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/nodestore/Database.cpp)\n\n###### Function:\n`Database::fetchNodeObject`", "type": "timeseries", "gridPos": { "x": 12, @@ -564,7 +564,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / ignoring(metric) rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Cache Hit Ratio\", \"\", \"\")", + "expr": "label_replace(rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / ignoring(metric) rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval]), \"series\", \"Found Ratio\", \"\", \"\")", "refId": "A" } ], @@ -573,7 +573,7 @@ "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "percentunit", "custom": { - "axisLabel": "Operations / Sec", + "axisLabel": "Found Ratio", "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, @@ -1347,8 +1347,8 @@ } }, { - "title": "NuDB Storage Size", - "description": "###### What this is:\n*On-disk size of the NuDB object-store back end, in bytes.*\n\n###### How it's computed:\n*Current value of the NuDB storage-size gauge, plotted over time.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the data growth rate.*\n\n###### Healthy range:\n*Gradual growth consistent with retained ledger history.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`", + "title": "NuDB Stored Bytes", + "description": "###### What this is:\n*Cumulative object-payload bytes this process has written to the NuDB object-store back end. Despite the metric name, this is not the size of the store on disk.*\n\n###### How it's computed:\n*Current value of the nudb_bytes gauge, plotted over time. It observes getStoreSize(), the same accessor node_written_bytes uses, so the two series are equal and their ratio is a constant 1.0 rather than a write-amplification measure.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the write rate. It excludes NuDB's keys, bucket padding and log, and it restarts from zero with the process while the files on disk do not.*\n\n###### Healthy range:\n*Gradual growth consistent with ledger data being stored.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill. Do not use this panel to size the store on disk or to plan disk capacity; no metric reports on-disk size today, so check the filesystem directly.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`", "type": "timeseries", "gridPos": { "h": 8, @@ -1368,7 +1368,7 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(storage_detail{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"nudb_bytes\"}, \"series\", \"NuDB Size\", \"\", \"\")" + "expr": "label_replace(storage_detail{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"nudb_bytes\"}, \"series\", \"NuDB Stored Bytes\", \"\", \"\")" } ], "fieldConfig": { @@ -1376,7 +1376,7 @@ "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "decbytes", "custom": { - "axisLabel": "Size", + "axisLabel": "Stored Bytes", "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10,